Upload 5 files

Dockerfile

@@ -0,0 +1,46 @@
+FROM python:3.9-slim
+
+WORKDIR /app
+
+# Install only essential dependencies that definitely exist in Debian Trixie
+RUN apt-get update && apt-get install -y \
+    wget \
+    curl \
+    libnss3 \
+    libatk-bridge2.0-0 \
+    libx11-6 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    libxss1 \
+    libasound2 \
+    libgtk-3-0 \
+    libgbm1 \
+    libcups2 \
+    libdbus-1-3 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements and install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Install Playwright with minimal dependencies
+RUN playwright install chromium
+
+# Copy application files
+COPY . .
+
+# Create necessary directories
+RUN mkdir -p /tmp
+
+# Expose port
+EXPOSE 7860
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:7860/health || exit 1
+
+# Start the application
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

ai_visibility.py

@@ -0,0 +1,1463 @@
+"""
+AI Visibility / AI Search Readiness analysis.
+
+Estimates how well a page is structured to be understood, retrieved, cited,
+summarized and recommended by AI-powered search systems (ChatGPT, Google AI
+Overviews, Perplexity, Claude, etc). These are NOT official ranking factors of
+any AI system - they are deterministic, measurable proxy signals derived from
+the page's HTML/text plus (optionally) a lightweight LLM pass for the handful
+of metrics that cannot be reliably computed with parsing alone.
+
+This module reports an "AI Readiness Score" - whether a page is technically
+and semantically prepared for AI systems. It does NOT claim to measure actual
+observed visibility in AI answers/citations (query coverage, citation rate,
+etc) - that requires live AI-query data this crawler does not have, so that
+field is always returned as "not_measured" rather than guessed at.
+
+Key design principles (see README/PR notes for the full rationale):
+  - Pages are classified by type (homepage, article, product, ...) and only
+    scored against metrics that are relevant to that type.
+  - Missing evidence is reported as "unknown" / null, never coerced to a low
+    score. Unknown != bad.
+  - Weights are centralized and configurable, per page type.
+
+This module is intentionally decoupled from seo_analyzer.py's SEO scoring.
+It reuses the same crawling primitives (discover_urls_parallel /
+fetch_all_pages_parallel) so pages are only fetched once per run via
+Playwright, but it does its own HTML parsing and its own scoring - it never
+touches or overrides the existing seo_score / page_summary fields.
+"""
+
+import os
+import re
+import json
+import asyncio
+from collections import Counter
+from urllib.parse import urlparse
+from urllib.request import urlopen, Request
+from dotenv import load_dotenv
+load_dotenv()
+
+from bs4 import BeautifulSoup
+
+from seo_analyzer import (
+    discover_urls_parallel,
+    fetch_all_pages_parallel,
+    OPENAI_AVAILABLE,
+)
+
+try:
+    import openai
+except Exception:
+    openai = None
+
+# SECURITY FIX: never hardcode API keys in source. Load from environment.
+# Set this in your shell / deployment config, e.g.:
+#   export OPENAI_API_KEY="sk-..."
+# If it's unset, `use_ai=True` calls will simply fall back to the
+# deterministic-only scoring path (see _llm_semantic_enhance below).
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+# Reused across calls so we're not re-instantiating the client every request.
+_openai_client = None
+
+
+def _get_openai_client():
+    global _openai_client
+    if _openai_client is None and openai is not None and OPENAI_API_KEY:
+        _openai_client = openai.OpenAI(api_key=OPENAI_API_KEY)
+    return _openai_client
+
+
+# ==============================
+# PAGE TYPES
+# ==============================
+PAGE_TYPES = [
+    "homepage", "portal", "article", "blog_post", "news", "product",
+    "product_category", "documentation", "forum", "review", "comparison",
+    "organization", "landing_page", "service_page", "directory",
+    "search_page", "unknown",
+]
+
+# ==============================
+# CENTRALIZED SCORING CONFIGURATION
+# ==============================
+# Base ("default") weights - used for page types without a specific profile.
+# Categories are the 8 "AI Readiness" pillars. Weights sum to 1.0.
+DEFAULT_WEIGHTS = {
+    "semantic": 0.20,
+    "content_answerability": 0.25,
+    "entity": 0.15,
+    "trust": 0.15,
+    "citation": 0.10,
+    "retrieval": 0.05,
+    "structured_data": 0.05,
+    "freshness": 0.05,
+}
+
+# Per-page-type overrides. Only categories that differ from DEFAULT_WEIGHTS
+# need to be listed - the profile is merged over the default and renormalized.
+PAGE_TYPE_WEIGHT_OVERRIDES = {
+    "homepage": {
+        "semantic": 0.15, "content_answerability": 0.10, "entity": 0.25,
+        "trust": 0.15, "citation": 0.05, "retrieval": 0.20,
+        "structured_data": 0.05, "freshness": 0.05,
+    },
+    "portal": {
+        "semantic": 0.15, "content_answerability": 0.10, "entity": 0.25,
+        "trust": 0.15, "citation": 0.05, "retrieval": 0.20,
+        "structured_data": 0.05, "freshness": 0.05,
+    },
+    "article": {
+        "semantic": 0.20, "content_answerability": 0.25, "entity": 0.10,
+        "trust": 0.15, "citation": 0.15, "retrieval": 0.05,
+        "structured_data": 0.05, "freshness": 0.05,
+    },
+    "blog_post": {
+        "semantic": 0.20, "content_answerability": 0.25, "entity": 0.10,
+        "trust": 0.15, "citation": 0.15, "retrieval": 0.05,
+        "structured_data": 0.05, "freshness": 0.05,
+    },
+    "news": {
+        "semantic": 0.18, "content_answerability": 0.22, "entity": 0.12,
+        "trust": 0.15, "citation": 0.13, "retrieval": 0.05,
+        "structured_data": 0.05, "freshness": 0.10,
+    },
+    "documentation": {
+        "semantic": 0.15, "content_answerability": 0.30, "entity": 0.10,
+        "trust": 0.10, "citation": 0.10, "retrieval": 0.10,
+        "structured_data": 0.10, "freshness": 0.05,
+    },
+    "product": {
+        "semantic": 0.10, "content_answerability": 0.20, "entity": 0.15,
+        "trust": 0.10, "citation": 0.05, "retrieval": 0.10,
+        "structured_data": 0.25, "freshness": 0.05,
+    },
+    "product_category": {
+        "semantic": 0.10, "content_answerability": 0.15, "entity": 0.15,
+        "trust": 0.10, "citation": 0.05, "retrieval": 0.15,
+        "structured_data": 0.25, "freshness": 0.05,
+    },
+    "forum": {
+        "semantic": 0.15, "content_answerability": 0.20, "entity": 0.10,
+        "trust": 0.15, "citation": 0.15, "retrieval": 0.10,
+        "structured_data": 0.05, "freshness": 0.10,
+    },
+    "review": {
+        "semantic": 0.15, "content_answerability": 0.20, "entity": 0.10,
+        "trust": 0.20, "citation": 0.15, "retrieval": 0.05,
+        "structured_data": 0.05, "freshness": 0.10,
+    },
+    "comparison": {
+        "semantic": 0.18, "content_answerability": 0.22, "entity": 0.12,
+        "trust": 0.15, "citation": 0.13, "retrieval": 0.05,
+        "structured_data": 0.05, "freshness": 0.10,
+    },
+    "organization": {
+        "semantic": 0.15, "content_answerability": 0.10, "entity": 0.25,
+        "trust": 0.25, "citation": 0.05, "retrieval": 0.10,
+        "structured_data": 0.05, "freshness": 0.05,
+    },
+    "service_page": {
+        "semantic": 0.15, "content_answerability": 0.15, "entity": 0.20,
+        "trust": 0.20, "citation": 0.05, "retrieval": 0.10,
+        "structured_data": 0.10, "freshness": 0.05,
+    },
+    "landing_page": {
+        "semantic": 0.15, "content_answerability": 0.10, "entity": 0.20,
+        "trust": 0.20, "citation": 0.05, "retrieval": 0.15,
+        "structured_data": 0.10, "freshness": 0.05,
+    },
+}
+
+
+def get_weights_for_page_type(page_type):
+    profile = dict(DEFAULT_WEIGHTS)
+    profile.update(PAGE_TYPE_WEIGHT_OVERRIDES.get(page_type, {}))
+    total = sum(profile.values()) or 1.0
+    return {k: v / total for k, v in profile.items()}
+
+
+# Per-page-type relevance map: which "trust/answerability" sub-signals are
+# actually meaningful for this page type. Signals not listed are treated as
+# not_applicable (excluded from scoring) rather than penalized when absent.
+TYPES_WHERE_AUTHOR_RELEVANT = {
+    "article", "blog_post", "news", "review", "comparison", "documentation", "forum",
+}
+TYPES_WHERE_FRESHNESS_RELEVANT = {
+    "article", "blog_post", "news", "review", "comparison", "documentation", "forum", "product",
+}
+TYPES_WHERE_FAQ_RELEVANT = {
+    "article", "blog_post", "documentation", "product", "product_category", "service_page", "landing_page",
+}
+TYPES_WHERE_PRODUCT_SCHEMA_RELEVANT = {"product", "product_category"}
+TYPES_WHERE_ARTICLE_SCHEMA_RELEVANT = {"article", "blog_post", "news"}
+
+DATE_PATTERNS = [
+    re.compile(r'\b(19|20)\d{2}-\d{2}-\d{2}\b'),
+    re.compile(r'\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+(19|20)\d{2}\b', re.I),
+    re.compile(r'\b\d{1,2}\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+(19|20)\d{2}\b', re.I),
+]
+
+FIRST_HAND_PATTERNS = re.compile(
+    r'\b(we tested|we found|our (research|study|testing|analysis|experiment)|in our experience|hands[- ]on|i tested|i used|we measured|we surveyed)\b',
+    re.I,
+)
+
+SOURCE_ATTRIBUTION_PATTERNS = re.compile(
+    r'\b(according to|source:|cited by|reported by|study by|research (from|by)|as reported)\b', re.I
+)
+
+
+def _words(text):
+    return re.findall(r"[A-Za-z']+", text or "")
+
+
+def _pct(n, d):
+    return round((n / d) * 100, 1) if d else 0.0
+
+
+def _clamp(v, lo=0, hi=100):
+    return max(lo, min(hi, v))
+
+
+# ==============================
+# PAGE TYPE CLASSIFICATION (deterministic)
+# ==============================
+def classify_page_type(seo_data, soup, text, word_count, schema_types, links):
+    """Classify the page using URL structure, schema, and content-structure
+    signals. Returns {"type": str, "confidence": float, "signals": [...]}.
+    No LLM call - this must be fast and run for every page.
+    """
+    url = seo_data.get("url", "")
+    path = urlparse(url).path.strip("/").lower()
+    lower_types = [t.lower() for t in schema_types]
+
+    heading_tags = soup.find_all(re.compile("^h[1-6]$"))
+    question_headings = sum(1 for h in heading_tags if h.get_text(strip=True).endswith("?"))
+    breadcrumb_present = bool(soup.find(attrs={"class": re.compile("breadcrumb", re.I)})) or any(
+        "breadcrumblist" in t for t in lower_types
+    )
+    has_price = bool(re.search(r'(\$|USD|EUR|£|₹)\s?\d', text or ""))
+    add_to_cart = bool(re.search(r'add to cart|buy now|add to bag|add to trolley', text or "", re.I))
+    nav_link_count = len(links)
+
+    votes = Counter()
+    signals = []
+
+    def vote(t, n, reason):
+        votes[t] += n
+        signals.append(f"{t}+{n}:{reason}")
+
+    # --- URL structure signals ---
+    if path == "":
+        vote("homepage", 3, "root path")
+    if re.search(r'\b(blog|article|post)\b', path):
+        vote("blog_post", 2, "url path")
+    if re.search(r'\bnews\b', path):
+        vote("news", 2, "url path")
+    if re.search(r'\b(docs?|documentation|guide|help|kb|support|wiki)\b', path):
+        vote("documentation", 2, "url path")
+    if re.search(r'\b(product|item|shop|store)\b', path):
+        vote("product", 2, "url path")
+    if re.search(r'\b(category|collection|catalog|categories)\b', path):
+        vote("product_category", 2, "url path")
+    if re.search(r'\b(forum|thread|topic|community|discussion)\b', path):
+        vote("forum", 2, "url path")
+    if re.search(r'\breview', path):
+        vote("review", 2, "url path")
+    if re.search(r'\b(vs|compare|comparison|alternatives)\b', path):
+        vote("comparison", 2, "url path")
+    if re.search(r'\b(about|company|who-we-are|team)\b', path):
+        vote("organization", 2, "url path")
+    if re.search(r'\b(pricing|services|solutions|features)\b', path):
+        vote("service_page", 1, "url path")
+    if re.search(r'\bsearch\b', path) or "q=" in urlparse(url).query:
+        vote("search_page", 2, "url/query")
+    if re.search(r'\b(directory|listings?)\b', path):
+        vote("directory", 1, "url path")
+    if re.search(r'\b(landing|lp)\b', path):
+        vote("landing_page", 1, "url path")
+
+    # --- schema.org signals ---
+    if "product" in lower_types:
+        vote("product", 3, "Product schema")
+    if any(t in lower_types for t in ("article", "newsarticle", "blogposting")):
+        vote("article" if "newsarticle" not in lower_types else "news", 3, "Article-family schema")
+    if "organization" in lower_types and path == "":
+        vote("organization", 1, "Organization schema on root")
+    if "faqpage" in lower_types:
+        vote("documentation", 1, "FAQPage schema")
+    if any(t in lower_types for t in ("itemlist", "collectionpage")):
+        vote("portal", 2, "ItemList/CollectionPage schema")
+    if "webpage" in lower_types and path == "":
+        vote("homepage", 1, "WebPage schema on root")
+
+    # --- structural/content signals ---
+    if nav_link_count >= 40 and word_count < 800:
+        vote("portal", 2, "high link density, low unique text")
+    if path == "" and nav_link_count >= 25:
+        vote("homepage", 2, "root path with heavy navigation")
+    if word_count >= 500 and heading_tags and not has_price:
+        vote("article", 1, "substantial prose content")
+    if has_price and add_to_cart:
+        vote("product", 3, "price + add-to-cart")
+    elif has_price:
+        vote("product", 1, "price present")
+    if question_headings >= 3:
+        vote("documentation", 1, "multiple question-style headings")
+        vote("forum", 1, "multiple question-style headings")
+    if breadcrumb_present and word_count > 300:
+        vote("article", 1, "breadcrumb + substantial content")
+
+    if not votes:
+        return {"type": "unknown", "confidence": 0.3, "signals": []}
+
+    page_type, top_votes = votes.most_common(1)[0]
+    total_votes = sum(votes.values())
+    confidence = round(min(0.98, 0.35 + (top_votes / max(1, total_votes)) * 0.6), 2)
+    return {"type": page_type, "confidence": confidence, "signals": signals}
+
+
+# ==============================
+# CATEGORY 1: TOPIC & SEMANTIC UNDERSTANDING (query-independent by default)
+# ==============================
+def _analyze_semantic(seo_data, soup, text, word_count, target_query=None):
+    title = (seo_data.get("title") or "").strip()
+    description = (seo_data.get("description") or "").strip()
+    h1s = [h.get_text(" ", strip=True) for h in soup.find_all("h1")]
+    h1_text = " ".join(h1s)
+
+    title_words = set(w.lower() for w in _words(title) if len(w) > 3)
+    h1_words = set(w.lower() for w in _words(h1_text) if len(w) > 3)
+    desc_words = set(w.lower() for w in _words(description) if len(w) > 3)
+
+    # topic_clarity: title exists, has an h1, and they share vocabulary.
+    # This does NOT require an external reference query.
+    topic_clarity = 0
+    if title:
+        topic_clarity += 40
+    if h1_text:
+        topic_clarity += 30
+    if title_words and h1_words:
+        overlap = len(title_words & h1_words) / max(1, len(title_words | h1_words))
+        topic_clarity += round(overlap * 30)
+    topic_clarity = _clamp(topic_clarity)
+
+    # topic_coherence: does title/description/h1 vocabulary agree with itself
+    # (internal consistency proxy, not "relevance" to any external query).
+    all_pairs = [p for p in [
+        (title_words, desc_words),
+        (title_words, h1_words),
+        (desc_words, h1_words),
+    ] if p[0] and p[1]]
+    if all_pairs:
+        sims = [len(a & b) / max(1, len(a | b)) for a, b in all_pairs]
+        topic_coherence = _clamp(round(sum(sims) / len(sims) * 100))
+    else:
+        topic_coherence = 0
+
+    # content_completeness: word count + heading coverage + list/table presence
+    heading_count = len(soup.find_all(re.compile("^h[1-6]$")))
+    completeness = 0
+    if word_count >= 1000:
+        completeness += 40
+    elif word_count >= 500:
+        completeness += 30
+    elif word_count >= 300:
+        completeness += 15
+    if heading_count >= 3:
+        completeness += 30
+    elif heading_count >= 1:
+        completeness += 15
+    if soup.find_all(["ul", "ol", "table"]):
+        completeness += 15
+    if description:
+        completeness += 15
+    content_completeness = _clamp(completeness)
+
+    # content_depth: vocabulary richness + paragraph count
+    paragraphs = [p.get_text(" ", strip=True) for p in soup.find_all("p")]
+    paragraphs = [p for p in paragraphs if p]
+    unique_words = set(w.lower() for w in _words(text))
+    richness = _pct(len(unique_words), max(1, word_count))
+    depth = 0
+    depth += min(40, round(richness))
+    depth += min(30, len(paragraphs) * 2)
+    depth += min(30, heading_count * 5)
+    content_depth = _clamp(depth)
+
+    result = {
+        "topic_clarity": topic_clarity,
+        "topic_coherence": topic_coherence,
+        "content_completeness": content_completeness,
+        "content_depth": content_depth,
+        "semantic_relevance": None,
+        "semantic_relevance_status": "requires_target_query",
+        "search_intent_match": None,
+        "search_intent_match_status": "requires_target_query",
+    }
+
+    # These two metrics genuinely require a reference query/keyword to mean
+    # anything. Without one they are reported as unknown, not guessed at.
+    if target_query:
+        query_words = set(w.lower() for w in _words(target_query) if len(w) > 2)
+        if query_words:
+            corpus_pairs = [
+                (query_words, title_words),
+                (query_words, h1_words),
+                (query_words, set(w.lower() for w in _words(text)[:400])),
+            ]
+            sims = [len(a & b) / max(1, len(a)) for a, b in corpus_pairs if a]
+            semantic_relevance = _clamp(round((sum(sims) / len(sims)) * 100)) if sims else 0
+            result["semantic_relevance"] = semantic_relevance
+            result["semantic_relevance_status"] = "measured"
+
+            intent_markers = ["how to", "what is", "why", "best", "guide", "review", "vs", "top ", "buy", "price"]
+            lowered_query = target_query.lower()
+            matched_intent = next((m for m in intent_markers if m in lowered_query), None)
+            overlap_in_content = len(query_words & set(w.lower() for w in _words(text))) / max(1, len(query_words))
+            search_intent_match = _clamp(round(overlap_in_content * 100))
+            if matched_intent in ("how to", "guide") and soup.find_all(["ol", "ul"]):
+                search_intent_match = _clamp(search_intent_match + 15)
+            result["search_intent_match"] = search_intent_match
+            result["search_intent_match_status"] = "measured"
+
+    return result
+
+
+# ==============================
+# CATEGORY 2: ENTITY UNDERSTANDING
+# ==============================
+def _analyze_entities(seo_data, soup, text, word_count, schema_types, page_type):
+    # Proper-noun style heuristic: capitalized word sequences not at sentence start.
+    # This works from visible text/headings/title/links - it does NOT require
+    # JSON-LD to detect entities. Schema (below) only boosts confidence.
+    proper_noun_seqs = re.findall(r'(?<!\. )(?<!^)\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3})\b', text or "")
+    title_seqs = re.findall(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3})\b', seo_data.get("title") or "")
+    heading_seqs = []
+    for h in soup.find_all(re.compile("^h[1-6]$")):
+        heading_seqs.extend(re.findall(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3})\b', h.get_text(" ", strip=True)))
+
+    entity_counter = Counter(s.strip() for s in proper_noun_seqs + title_seqs + heading_seqs if len(s.strip()) > 2)
+    entity_count = len(entity_counter)
+
+    if word_count < 20:
+        entity_clarity = None
+        entity_clarity_status = "insufficient_text"
+    else:
+        entity_clarity = _clamp(round(_pct(entity_count, max(1, word_count // 20))))
+        entity_clarity_status = "measured"
+
+    if entity_counter:
+        top_mentions = entity_counter.most_common(1)[0][1]
+        entity_consistency = _clamp(round(_pct(top_mentions, sum(entity_counter.values()))))
+    else:
+        entity_consistency = 0
+
+    lower_types = [t.lower() for t in schema_types]
+    organization_entity_present = bool(
+        {"organization", "corporation", "localbusiness"} & set(lower_types)
+    ) or bool(soup.find(attrs={"itemtype": re.compile("Organization", re.I)})) or bool(
+        soup.find("footer") and re.search(r'\b(Inc\.|LLC|Ltd\.|Corporation|Corp\.)\b', text or "")
+    )
+
+    author_entity_present = bool(
+        seo_data.get("metas", {}).get("author")
+    ) or bool(soup.find(attrs={"rel": "author"})) or bool(
+        soup.find(class_=re.compile("author|byline", re.I))
+    ) or "person" in lower_types
+
+    product_entity_present = "product" in lower_types or bool(
+        re.search(r'\b(SKU|model number|add to cart)\b', text or "", re.I)
+    )
+    place_entity_present = bool(re.search(r'\b\d{5}(-\d{4})?\b', text or "")) and bool(
+        re.search(r'\b(Street|Ave|Avenue|Road|Blvd|City|Country)\b', text or "", re.I)
+    )
+    brand_entity_present = organization_entity_present or bool(
+        re.search(r'\b(®|™)\b', text or "")
+    )
+
+    entity_types_present = [
+        t for t, present in [
+            ("organization", organization_entity_present),
+            ("person", author_entity_present),
+            ("product", product_entity_present),
+            ("place", place_entity_present),
+            ("brand", brand_entity_present),
+        ] if present
+    ]
+
+    return {
+        "entity_count": entity_count,
+        "entity_clarity": entity_clarity,
+        "entity_clarity_status": entity_clarity_status,
+        "entity_consistency": entity_consistency,
+        "entity_types": entity_types_present,
+        "organization_entity_present": organization_entity_present,
+        "author_entity_present": author_entity_present,
+        "product_entity_present": product_entity_present,
+        "place_entity_present": place_entity_present,
+        "brand_entity_present": brand_entity_present,
+    }
+
+
+# ==============================
+# CATEGORY 3: ANSWERABILITY
+# ==============================
+def _analyze_answerability(soup, text):
+    heading_tags = soup.find_all(re.compile("^h[1-6]$"))
+    question_headings = [h for h in heading_tags if h.get_text(strip=True).endswith("?")]
+    question_count = len(question_headings)
+
+    def _next_text_len(tag):
+        sib = tag.find_next_sibling()
+        hops = 0
+        while sib is not None and hops < 3:
+            content = sib.get_text(" ", strip=True)
+            if content:
+                return len(_words(content))
+            sib = sib.find_next_sibling()
+            hops += 1
+        return 0
+
+    answered = sum(1 for h in question_headings if _next_text_len(h) >= 15)
+    questions_answered = answered
+    answer_coverage = _pct(answered, question_count) if question_count else None
+
+    first_p = soup.find("p")
+    first_p_text = first_p.get_text(" ", strip=True) if first_p else ""
+    direct_answer_presence = bool(first_p_text) and 40 <= len(first_p_text) <= 400
+
+    definition_presence = bool(
+        re.search(r'\b\w+\s+(is|are|refers to|means)\s+(a|an|the)\b', text or "", re.I)
+    ) or bool(soup.find("dfn")) or bool(soup.find("dl"))
+
+    faq_heading = soup.find(
+        lambda t: t.name in ("h1", "h2", "h3") and re.search(r"faq|frequently asked", t.get_text(" ", strip=True), re.I)
+    )
+    faq_coverage = 100 if faq_heading else (_clamp(question_count * 20) if question_count else 0)
+
+    return {
+        "question_count": question_count,
+        "questions_answered": questions_answered,
+        "answer_coverage": answer_coverage,
+        "direct_answer_presence": direct_answer_presence,
+        "definition_presence": definition_presence,
+        "faq_coverage": faq_coverage,
+    }
+
+
+# ==============================
+# CATEGORY 4: INFORMATION QUALITY
+# ==============================
+def _analyze_information_quality(soup, text):
+    sentences = re.split(r'(?<=[.!?])\s+', text or "")
+    stat_pattern = re.compile(r'\b\d+([.,]\d+)?\s?(%|percent)?\b')
+    factual_sentences = [s for s in sentences if re.search(r'\d', s) and stat_pattern.search(s)]
+    factual_claims = len(factual_sentences)
+
+    claims_with_sources = sum(
+        1 for s in factual_sentences if SOURCE_ATTRIBUTION_PATTERNS.search(s)
+    )
+    claims_with_sources_ratio = _pct(claims_with_sources, max(1, factual_claims)) if factual_claims else None
+
+    original_info_hits = len(FIRST_HAND_PATTERNS.findall(text or ""))
+    original_information = _clamp(min(100, original_info_hits * 25))
+    first_hand_experience = original_information
+
+    return {
+        "factual_claims": factual_claims,
+        "claims_with_sources": claims_with_sources,
+        "claims_with_sources_ratio": claims_with_sources_ratio,
+        "original_information": original_information,
+        "first_hand_experience": first_hand_experience,
+    }
+
+
+# ==============================
+# CATEGORY 5: TRUST / AUTHORITY
+# ==============================
+def _analyze_trust(seo_data, soup, text, links, page_type):
+    author_relevant = page_type in TYPES_WHERE_AUTHOR_RELEVANT
+
+    author_bio_hit = bool(re.search(r'\b(PhD|M\.?D\.?|certified|expert|years of experience|founder|CEO|specialist)\b', text or "", re.I))
+    if author_relevant:
+        author_expertise = 70 if author_bio_hit else 0
+        author_expertise_status = "measured"
+    else:
+        author_expertise = None
+        author_expertise_status = "not_applicable"
+    author_credentials = author_bio_hit
+
+    hrefs = [l.get("href", "") for l in links]
+    about_page_present = any(re.search(r'/about', h, re.I) for h in hrefs)
+    contact_page_present = any(re.search(r'/contact', h, re.I) for h in hrefs)
+    email_present = bool(re.search(r'[\w.+-]+@[\w-]+\.[\w.-]+', text or ""))
+    phone_present = bool(re.search(r'(\+?\d[\d\s().-]{7,}\d)', text or ""))
+    contact_information_present = contact_page_present or email_present or phone_present
+
+    privacy_present = any(re.search(r'privacy', h, re.I) for h in hrefs)
+    terms_present = any(re.search(r'terms', h, re.I) for h in hrefs)
+    https_present = str(seo_data.get("url", "")).startswith("https://")
+
+    trust_signal_flags = [about_page_present, contact_information_present, privacy_present, terms_present, https_present]
+    trust_signals = sum(trust_signal_flags)
+
+    organization_transparency = _clamp(_pct(trust_signals, len(trust_signal_flags)))
+
+    return {
+        "author_expertise": author_expertise,
+        "author_expertise_status": author_expertise_status,
+        "author_credentials": author_credentials if author_relevant else None,
+        "organization_transparency": organization_transparency,
+        "about_page_present": about_page_present,
+        "contact_information_present": contact_information_present,
+        "trust_signals": trust_signals,
+    }
+
+
+# ==============================
+# CATEGORY 6: STRUCTURED DATA (supporting signal, not dominant)
+# ==============================
+def _extract_schema_types(seo_data, soup):
+    schemas = seo_data.get("schemas", [])
+    schema_types = []
+    schema_valid = True
+
+    for schema in schemas:
+        try:
+            if isinstance(schema, dict):
+                if "@type" in schema:
+                    t = schema["@type"]
+                    schema_types.extend(t if isinstance(t, list) else [t])
+                if "@graph" in schema and isinstance(schema["@graph"], list):
+                    for item in schema["@graph"]:
+                        if isinstance(item, dict) and "@type" in item:
+                            schema_types.append(item["@type"])
+            elif isinstance(schema, list):
+                for item in schema:
+                    if isinstance(item, dict) and "@type" in item:
+                        schema_types.append(item["@type"])
+        except Exception:
+            schema_valid = False
+
+    schema_types = list(dict.fromkeys(str(t) for t in schema_types))
+    return schema_types, schema_valid
+
+
+def _analyze_structured_data(seo_data, schema_types, schema_valid, page_type):
+    lower_types = [t.lower() for t in schema_types]
+    schema_present = len(schema_types) > 0
+    organization_schema = any("organization" in t for t in lower_types)
+    article_schema = any("article" in t for t in lower_types)
+    product_schema = any("product" in t for t in lower_types)
+    faq_schema = any("faq" in t for t in lower_types)
+    breadcrumb_schema = any("breadcrumb" in t for t in lower_types)
+
+    # completeness: how many of the schema types *relevant to this page type*
+    # are present, rather than expecting every type on every page.
+    relevant_types = {"organization"}
+    if page_type in TYPES_WHERE_ARTICLE_SCHEMA_RELEVANT:
+        relevant_types.add("article")
+    if page_type in TYPES_WHERE_PRODUCT_SCHEMA_RELEVANT:
+        relevant_types.add("product")
+    if page_type in TYPES_WHERE_FAQ_RELEVANT:
+        relevant_types.add("faq")
+    present_map = {
+        "organization": organization_schema, "article": article_schema,
+        "product": product_schema, "faq": faq_schema,
+    }
+    relevant_present = sum(1 for t in relevant_types if present_map.get(t))
+    schema_completeness = _clamp(round(_pct(relevant_present, max(1, len(relevant_types)))))
+
+    org_schema_name = None
+    for schema in seo_data.get("schemas", []) or []:
+        if isinstance(schema, dict) and str(schema.get("@type", "")).lower() == "organization":
+            org_schema_name = schema.get("name")
+            break
+    domain = urlparse(seo_data.get("url", "")).netloc.replace("www.", "").split(".")[0]
+    schema_entity_alignment = bool(
+        org_schema_name and domain and domain.lower() in str(org_schema_name).lower()
+    )
+
+    return {
+        "schema_present": schema_present,
+        "schema_types": ", ".join(schema_types) if schema_types else "not_detected",
+        "schema_valid": schema_valid,
+        "schema_completeness": schema_completeness,
+        "schema_entity_alignment": schema_entity_alignment,
+        "organization_schema": organization_schema,
+        "article_schema": article_schema,
+        "product_schema": product_schema,
+        "faq_schema": faq_schema,
+        "breadcrumb_schema": breadcrumb_schema,
+    }
+
+
+# ==============================
+# CATEGORY 7: RETRIEVAL / CRAWLABILITY
+# ==============================
+def _fetch_raw_html_sync(url, timeout=6):
+    """Cheap plain-HTTP GET (no browser) used only to compare against the
+    Playwright-rendered HTML, so we can tell whether critical content is
+    server-rendered or injected by JavaScript. This is a single lightweight
+    request per page, not a second full crawl."""
+    try:
+        req = Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; AIVisibilityBot/1.0)"})
+        with urlopen(req, timeout=timeout) as resp:
+            raw = resp.read(2_000_000)
+        return raw.decode("utf-8", errors="ignore")
+    except Exception:
+        return None
+
+
+async def _analyze_retrieval(seo_data, word_count, text, html):
+    robots = (seo_data.get("robots") or "").lower()
+    indexable = "noindex" not in robots
+    robots_allowed = indexable  # proxy: only meta robots is checked, robots.txt is not fetched
+
+    canonical = seo_data.get("canonical") or ""
+    domain = urlparse(seo_data.get("url", "")).netloc
+    canonical_valid = bool(canonical) and (domain == "" or domain in canonical)
+
+    content_accessible = word_count > 0
+    renderable_content = word_count >= 50
+
+    url = seo_data.get("url", "")
+    raw_html = None
+    if url:
+        raw_html = await asyncio.get_event_loop().run_in_executor(None, _fetch_raw_html_sync, url)
+
+    content_rendering = {
+        "raw_word_count": None,
+        "rendered_word_count": word_count,
+        "rendering_dependency_ratio": None,
+        "critical_content_server_rendered": None,
+        "retrieval_status": "unknown",
+    }
+
+    if raw_html is not None:
+        raw_soup = BeautifulSoup(raw_html, "html.parser")
+        raw_text = raw_soup.get_text(separator=" ", strip=True)
+        raw_word_count = len(_words(raw_text))
+        content_rendering["raw_word_count"] = raw_word_count
+        ratio = _pct(raw_word_count, max(1, word_count))
+        content_rendering["rendering_dependency_ratio"] = ratio
+        content_rendering["critical_content_server_rendered"] = raw_word_count >= 50 and ratio >= 60
+        if not content_accessible:
+            content_rendering["retrieval_status"] = "poor"
+        elif content_rendering["critical_content_server_rendered"]:
+            content_rendering["retrieval_status"] = "good"
+        elif raw_word_count >= 50:
+            content_rendering["retrieval_status"] = "limited"
+        else:
+            content_rendering["retrieval_status"] = "js_dependent"
+    else:
+        # Couldn't do the plain-HTTP comparison (blocked, timeout, etc).
+        # We do NOT penalize the page for this - it's a measurement gap.
+        content_rendering["retrieval_status"] = "unknown"
+
+    return {
+        "indexable": indexable,
+        "robots_allowed": robots_allowed,
+        "canonical_valid": canonical_valid,
+        "content_accessible": content_accessible,
+        "renderable_content": renderable_content,
+        "content_rendering": content_rendering,
+    }
+
+
+# ==============================
+# CATEGORY 8: CONTENT STRUCTURE
+# ==============================
+def _analyze_content_structure(soup):
+    heading_tags = soup.find_all(re.compile("^h[1-6]$"))
+    levels = []
+    for h in heading_tags:
+        try:
+            levels.append(int(h.name[1]))
+        except Exception:
+            continue
+
+    heading_structure_score = 0
+    if 1 in levels:
+        heading_structure_score += 40
+    if 2 in levels:
+        heading_structure_score += 30
+    if levels == sorted(levels):
+        heading_structure_score += 30
+    heading_structure_score = _clamp(heading_structure_score)
+
+    paragraphs = [p.get_text(" ", strip=True) for p in soup.find_all("p")]
+    paragraphs = [p for p in paragraphs if p]
+    if paragraphs:
+        avg_len = sum(len(_words(p)) for p in paragraphs) / len(paragraphs)
+        paragraph_clarity = 100 if 15 <= avg_len <= 40 else _clamp(100 - abs(avg_len - 27) * 3)
+    else:
+        paragraph_clarity = 0
+
+    list_usage = len(soup.find_all(["ul", "ol"])) > 0
+    table_usage = len(soup.find_all("table")) > 0
+    definition_sections = bool(soup.find("dl")) or bool(soup.find("dfn"))
+    summary_present = bool(soup.find(
+        lambda t: t.name in ("h1", "h2", "h3") and re.search(r"summary|tl;?dr|key takeaways", t.get_text(" ", strip=True), re.I)
+    ))
+
+    return {
+        "heading_structure_score": heading_structure_score,
+        "paragraph_clarity": round(paragraph_clarity),
+        "list_usage": list_usage,
+        "table_usage": table_usage,
+        "definition_sections": definition_sections,
+        "summary_present": summary_present,
+    }
+
+
+# ==============================
+# CATEGORY 9: CITATION POTENTIAL
+# (based on original data/quotes/attribution actually present in the
+# content, not on raw external-link counting)
+# ==============================
+def _analyze_citation_potential(soup, text):
+    numbers = re.findall(r'\b\d+(?:[.,]\d+)?%?\b', text or "")
+    unique_data_points = len(set(numbers))
+    statistics_present = unique_data_points > 0
+
+    original_research = bool(FIRST_HAND_PATTERNS.search(text or ""))
+
+    blockquotes = soup.find_all("blockquote")
+    quoted_sentences = re.findall(r'"[^"]{20,200}"', text or "")
+    quotable_statements = len(blockquotes) + len(quoted_sentences)
+
+    source_attribution = len(SOURCE_ATTRIBUTION_PATTERNS.findall(text or ""))
+
+    score = 0
+    score += min(35, unique_data_points * 3)
+    score += 25 if original_research else 0
+    score += min(20, quotable_statements * 5)
+    score += min(20, source_attribution * 10)
+    citation_potential = _clamp(score)
+
+    return {
+        "unique_data_points": unique_data_points,
+        "statistics_present": statistics_present,
+        "original_research": original_research,
+        "quotable_statements": quotable_statements,
+        "source_attribution": source_attribution,
+        "citation_potential": citation_potential,
+    }
+
+
+# ==============================
+# CATEGORY 10: FRESHNESS (fresh / stale / very_stale / unknown - never
+# "guessed outdated")
+# ==============================
+def _analyze_freshness(seo_data, soup, text, page_type):
+    metas = seo_data.get("metas", {}) or {}
+    last_updated = (
+        metas.get("article:modified_time")
+        or metas.get("article:published_time")
+        or metas.get("date")
+    )
+
+    if not last_updated:
+        time_tag = soup.find("time", attrs={"datetime": True})
+        if time_tag:
+            last_updated = time_tag.get("datetime")
+
+    if not last_updated:
+        for pattern in DATE_PATTERNS:
+            match = pattern.search(text or "")
+            if match:
+                last_updated = match.group(0)
+                break
+
+    date_visible = bool(last_updated)
+    content_age_days = None
+
+    if last_updated:
+        try:
+            from datetime import datetime, timezone
+            parsed = None
+            for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
+                try:
+                    parsed = datetime.strptime(last_updated[:19].replace("Z", ""), fmt)
+                    break
+                except Exception:
+                    continue
+            if parsed:
+                parsed = parsed.replace(tzinfo=timezone.utc)
+                content_age_days = (datetime.now(timezone.utc) - parsed).days
+        except Exception:
+            content_age_days = None
+
+    if not date_visible:
+        freshness_status = "unknown"
+        freshness_reason = "No reliable publication/update date detected on the page."
+    elif isinstance(content_age_days, int):
+        # FIX: previously the 365< age <=730 and >730 branches both produced
+        # "stale", making the 730-day boundary dead code. Now a genuine
+        # "very_stale" tier exists for content older than 2 years.
+        if content_age_days <= 365:
+            freshness_status = "fresh"
+        elif content_age_days <= 730:
+            freshness_status = "stale"
+        else:
+            freshness_status = "very_stale"
+        freshness_reason = f"Date detected: content is {content_age_days} days old."
+    else:
+        freshness_status = "unknown"
+        freshness_reason = "A date-like string was found but could not be reliably parsed."
+
+    return {
+        "last_updated": last_updated or "not_detected",
+        "content_age_days": content_age_days,
+        "update_frequency": "not_detected",  # requires historical crawl data, unavailable here
+        "date_visible": date_visible,
+        "freshness_status": freshness_status,
+        "freshness_reason": freshness_reason,
+        "freshness_relevant": page_type in TYPES_WHERE_FRESHNESS_RELEVANT,
+    }
+
+
+# ==============================
+# CATEGORY 11: BRAND / ENTITY CONSISTENCY
+# ==============================
+def _analyze_brand_consistency(seo_data, soup, text, schema_types):
+    domain = urlparse(seo_data.get("url", "")).netloc.replace("www.", "")
+    brand_guess = domain.split(".")[0] if domain else ""
+
+    title = (seo_data.get("title") or "")
+    footer = soup.find("footer")
+    footer_text = footer.get_text(" ", strip=True) if footer else ""
+
+    brand_in_title = bool(brand_guess) and brand_guess.lower() in title.lower()
+    brand_in_footer = bool(brand_guess) and brand_guess.lower() in footer_text.lower()
+    brand_in_body = bool(brand_guess) and brand_guess.lower() in (text or "").lower()
+    brand_name_consistency = _clamp(sum([brand_in_title, brand_in_footer, brand_in_body]) * 33)
+
+    org_schema_name = None
+    for schema in seo_data.get("schemas", []) or []:
+        if isinstance(schema, dict) and str(schema.get("@type", "")).lower() == "organization":
+            org_schema_name = schema.get("name")
+            break
+
+    company_information_consistency = _clamp(
+        70 if (org_schema_name and brand_guess and brand_guess.lower() in str(org_schema_name).lower()) else (30 if brand_in_footer else 0)
+    )
+
+    author_meta = (seo_data.get("metas", {}) or {}).get("author", "")
+    byline = soup.find(class_=re.compile("author|byline", re.I))
+    byline_text = byline.get_text(" ", strip=True) if byline else ""
+    author_information_consistency = _clamp(
+        70 if (author_meta and byline_text and author_meta.lower() in byline_text.lower())
+        else (40 if (author_meta or byline_text) else 0)
+    )
+
+    return {
+        "brand_name_consistency": brand_name_consistency,
+        "company_information_consistency": company_information_consistency,
+        "author_information_consistency": author_information_consistency,
+    }
+
+
+# ==============================
+# CATEGORY SCORE ROLLUPS (unknown values excluded, weight redistributed)
+# ==============================
+def _avg_known(*values):
+    """Average only the non-None values. Returns None if all are unknown -
+    the caller decides how to treat that (usually: exclude from rollup)."""
+    known = [v for v in values if v is not None]
+    if not known:
+        return None
+    return round(sum(known) / len(known))
+
+
+def _rollup_scores(m, page_type):
+    weights = get_weights_for_page_type(page_type)
+
+    semantic_parts = [m["topic_clarity"], m["topic_coherence"], m["content_completeness"], m["content_depth"]]
+    if m.get("semantic_relevance") is not None:
+        semantic_parts.append(m["semantic_relevance"])
+    if m.get("search_intent_match") is not None:
+        semantic_parts.append(m["search_intent_match"])
+    semantic = _avg_known(*semantic_parts)
+
+    content_answerability = _avg_known(
+        m["content_completeness"],
+        m["answer_coverage"],
+        100 if m["direct_answer_presence"] else 0,
+        100 if m["definition_presence"] else 0,
+        m["faq_coverage"] if page_type in TYPES_WHERE_FAQ_RELEVANT or m["question_count"] > 0 else None,
+        _clamp(min(100, m["factual_claims"] * 10)),
+    )
+
+    entity_parts = [m["entity_consistency"]]
+    if m.get("entity_clarity") is not None:
+        entity_parts.append(m["entity_clarity"])
+    entity_parts.append(100 if m["organization_entity_present"] else 0)
+    if page_type in TYPES_WHERE_AUTHOR_RELEVANT:
+        entity_parts.append(100 if m["author_entity_present"] else 0)
+    entity = _avg_known(*entity_parts)
+
+    trust_parts = [
+        m["organization_transparency"],
+        100 if m["about_page_present"] else 0,
+        100 if m["contact_information_present"] else 0,
+    ]
+    if m.get("author_expertise") is not None:
+        trust_parts.append(m["author_expertise"])
+    trust = _avg_known(*trust_parts)
+
+    citation = m["citation_potential"]
+
+    retrieval_parts = [
+        100 if m["indexable"] else 0,
+        100 if m["canonical_valid"] else 0,
+        100 if m["content_accessible"] else 0,
+    ]
+    rendering_status = m.get("content_rendering", {}).get("retrieval_status")
+    if rendering_status == "good":
+        retrieval_parts.append(100)
+    elif rendering_status == "limited":
+        retrieval_parts.append(60)
+    elif rendering_status == "js_dependent":
+        retrieval_parts.append(30)
+    # "unknown" contributes nothing - not penalized, not rewarded.
+    retrieval = _avg_known(*retrieval_parts)
+
+    schema_hits = sum([
+        m["organization_schema"], m["article_schema"], m["product_schema"],
+        m["faq_schema"], m["breadcrumb_schema"],
+    ])
+    structured_data = _clamp(
+        m["schema_completeness"] * 0.6 + (20 if m["schema_valid"] and m["schema_present"] else 0) + schema_hits * 4
+    )
+
+    if m["freshness_status"] == "fresh":
+        if isinstance(m["content_age_days"], int):
+            freshness = 100 if m["content_age_days"] <= 90 else 75
+        else:
+            freshness = 75
+    elif m["freshness_status"] == "stale":
+        freshness = 30
+    elif m["freshness_status"] == "very_stale":
+        freshness = 10
+    else:
+        freshness = None  # unknown - excluded from weighted rollup entirely
+
+    components = {
+        "semantic": semantic,
+        "content_answerability": content_answerability,
+        "entity": entity,
+        "trust": trust,
+        "citation": citation,
+        "retrieval": retrieval,
+        "structured_data": structured_data,
+        "freshness": freshness,
+    }
+
+    # Weighted average over KNOWN components only; unknown components'
+    # weight is redistributed proportionally rather than counted as 0.
+    known_weight = sum(weights[k] for k, v in components.items() if v is not None)
+    if known_weight <= 0:
+        ai_readiness_score = 0
+    else:
+        ai_readiness_score = round(
+            sum(components[k] * weights[k] for k in components if components[k] is not None) / known_weight
+        )
+    ai_readiness_score = _clamp(ai_readiness_score)
+
+    return {
+        "semantic_score": semantic,
+        "content_answerability_score": content_answerability,
+        "entity_score": entity,
+        "trust_score": trust,
+        "citation_potential_score": citation,
+        "retrieval_score": retrieval,
+        "structured_data_score": structured_data,
+        "freshness_score": freshness,
+        "ai_readiness_score": ai_readiness_score,
+        # legacy alias kept for the existing frontend/API consumers
+        "ai_visibility_score": ai_readiness_score,
+        "weights_used": weights,
+    }
+
+
+# ==============================
+# OPTIONAL LLM ENHANCEMENT (semantic-only, reuses existing OpenAI setup)
+# ==============================
+async def _llm_semantic_enhance(seo_data, text, deterministic, target_query=None):
+    # FIX: previously used the removed openai<1.0 `openai.ChatCompletion.create`
+    # API, which raises AttributeError on any openai-python>=1.0 install and
+    # silently fell back to deterministic-only scoring every time. Now uses
+    # the current client-based API (openai.OpenAI().chat.completions.create).
+    client = _get_openai_client()
+    if not OPENAI_AVAILABLE or client is None:
+        return None
+
+    excerpt = (text or "")[:2000]
+    query_line = f"TARGET QUERY: {target_query}\n" if target_query else ""
+    prompt = f"""You are assessing AI-search readiness of a web page (not traditional SEO).
+Given the page title, meta description and a content excerpt, score each item 0-100 based on how easily
+an AI system could understand and summarize this page. Return ONLY valid JSON with these exact keys:
+topic_clarity, content_completeness, entity_clarity, citation_potential, original_information{"," if target_query else ""}
+{"semantic_relevance, search_intent_match" if target_query else ""}
+
+{query_line}TITLE: {seo_data.get('title', '')}
+META DESCRIPTION: {seo_data.get('description', '')}
+CONTENT EXCERPT: {excerpt}
+"""
+    try:
+        response = await asyncio.get_event_loop().run_in_executor(
+            None,
+            lambda: client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[
+                    {"role": "system", "content": "You output only strict JSON, no prose, no markdown fences."},
+                    {"role": "user", "content": prompt},
+                ],
+                max_tokens=300,
+                temperature=0.3,
+            ),
+        )
+        raw = response.choices[0].message.content.strip()
+        raw = re.sub(r"^```(json)?|```$", "", raw.strip(), flags=re.I).strip()
+        data = json.loads(raw)
+        keys = ["topic_clarity", "content_completeness", "entity_clarity", "citation_potential", "original_information"]
+        if target_query:
+            keys += ["semantic_relevance", "search_intent_match"]
+        cleaned = {}
+        for k in keys:
+            v = data.get(k)
+            if isinstance(v, (int, float)):
+                cleaned[k] = _clamp(round(v))
+        return cleaned or None
+    except Exception as e:
+        print(f"   AI visibility LLM enhancement failed: {str(e)[:120]}")
+        return None
+
+
+def _blend(deterministic_value, llm_value):
+    if llm_value is None:
+        return deterministic_value
+    if deterministic_value is None:
+        return llm_value
+    return round((deterministic_value + llm_value) / 2)
+
+
+# ==============================
+# PER-PAGE ORCHESTRATION
+# ==============================
+async def analyze_page_ai_visibility(seo_data, domain, use_ai=False, target_query=None):
+    """Compute the full ai_visibility metric set for one already-fetched page."""
+    empty_scores = {
+        "semantic_score": None, "content_answerability_score": None, "entity_score": None,
+        "trust_score": None, "citation_potential_score": None, "retrieval_score": None,
+        "structured_data_score": None, "freshness_score": None,
+        "ai_readiness_score": 0, "ai_visibility_score": 0, "weights_used": {},
+    }
+    try:
+        html = seo_data.get("html", "")
+        soup = BeautifulSoup(html, "html.parser")
+        text = soup.get_text(separator=" ", strip=True)
+        word_count = len(_words(text))
+        links = seo_data.get("links", [])
+
+        schema_types, schema_valid = _extract_schema_types(seo_data, soup)
+        page_type_info = classify_page_type(seo_data, soup, text, word_count, schema_types, links)
+        page_type = page_type_info["type"]
+
+        m = {}
+        m.update(_analyze_semantic(seo_data, soup, text, word_count, target_query))
+        m.update(_analyze_entities(seo_data, soup, text, word_count, schema_types, page_type))
+        m.update(_analyze_answerability(soup, text))
+        m.update(_analyze_information_quality(soup, text))
+        m.update(_analyze_trust(seo_data, soup, text, links, page_type))
+        m.update(_analyze_structured_data(seo_data, schema_types, schema_valid, page_type))
+        m.update(await _analyze_retrieval(seo_data, word_count, text, html))
+        m.update(_analyze_content_structure(soup))
+        m.update(_analyze_citation_potential(soup, text))
+        m.update(_analyze_freshness(seo_data, soup, text, page_type))
+        m.update(_analyze_brand_consistency(seo_data, soup, text, schema_types))
+
+        if use_ai:
+            llm_result = await _llm_semantic_enhance(seo_data, text, m, target_query)
+            if llm_result:
+                for k, v in llm_result.items():
+                    if k in m:
+                        m[k] = _blend(m[k], v)
+
+        scores = _rollup_scores(m, page_type)
+
+        # Build unknown/not-applicable metric lists for transparency.
+        unknown_metrics = []
+        if m.get("semantic_relevance_status") == "requires_target_query":
+            unknown_metrics.append({"metric": "semantic_relevance", "reason": "No target keyword/query was supplied for this analysis."})
+        if m.get("freshness_status") == "unknown":
+            unknown_metrics.append({"metric": "freshness", "reason": m.get("freshness_reason", "No date detected.")})
+        if m.get("author_expertise_status") == "not_applicable":
+            unknown_metrics.append({"metric": "author_expertise", "reason": f"Not applicable for page type '{page_type}'."})
+        if m.get("content_rendering", {}).get("retrieval_status") == "unknown":
+            unknown_metrics.append({"metric": "retrieval_rendering", "reason": "Could not fetch raw HTML to compare against rendered content."})
+
+        page_ai_visibility = {
+            "page_type": page_type,
+            "page_type_confidence": page_type_info["confidence"],
+            "topic_clarity": m["topic_clarity"],
+            "semantic_relevance": m["semantic_relevance"],
+            "content_completeness": m["content_completeness"],
+            "entity_clarity": m["entity_clarity"],
+            "answer_coverage": m["answer_coverage"],
+            "factual_information": m["factual_claims"],
+            "original_information": m["original_information"],
+            "author_expertise": m["author_expertise"],
+            "schema_quality": scores["structured_data_score"],
+            "crawlability": scores["retrieval_score"],
+            "content_structure": m["heading_structure_score"],
+            "citation_potential": m["citation_potential"],
+            "freshness_status": m["freshness_status"],
+            "freshness": scores["freshness_score"],
+            "ai_readiness_score": scores["ai_readiness_score"],
+            "ai_visibility_score": scores["ai_visibility_score"],
+        }
+
+        return {
+            "url": seo_data.get("url", ""),
+            "title": seo_data.get("title", ""),
+            "page_type": page_type,
+            "page_type_confidence": page_type_info["confidence"],
+            "raw_metrics": m,
+            "scores": scores,
+            "unknown_metrics": unknown_metrics,
+            "ai_visibility": page_ai_visibility,
+        }
+    except Exception as e:
+        print(f"AI visibility analysis error for {seo_data.get('url', 'unknown')}: {e}")
+        return {
+            "url": seo_data.get("url", ""),
+            "title": seo_data.get("title", ""),
+            "page_type": "unknown",
+            "page_type_confidence": 0,
+            "raw_metrics": {},
+            "scores": empty_scores,
+            "unknown_metrics": [],
+            "ai_visibility": {
+                "page_type": "unknown", "page_type_confidence": 0,
+                "topic_clarity": 0, "semantic_relevance": None, "content_completeness": 0,
+                "entity_clarity": None, "answer_coverage": None, "factual_information": 0,
+                "original_information": 0, "author_expertise": None, "schema_quality": 0,
+                "crawlability": 0, "content_structure": 0, "citation_potential": 0,
+                "freshness_status": "unknown", "freshness": None,
+                "ai_readiness_score": 0, "ai_visibility_score": 0,
+            },
+            "error": str(e),
+        }
+
+
+# ==============================
+# ISSUE / STRENGTH GENERATION (evidence-based, page-type-aware)
+# ==============================
+def _build_issues_and_strengths(page_results):
+    issues = []
+    strengths = []
+    unknowns = []
+
+    for p in page_results:
+        s = p["scores"]
+        m = p["raw_metrics"]
+        url = p["url"]
+        page_type = p.get("page_type", "unknown")
+        if not m:
+            continue
+
+        # --- Answerability: only relevant where questions/FAQ genuinely matter ---
+        if page_type in TYPES_WHERE_FAQ_RELEVANT or m.get("question_count", 0) > 0:
+            ca_score = s.get("content_answerability_score")
+            if ca_score is not None and ca_score < 50:
+                issues.append({
+                    "title": "Missing clear answers to common questions",
+                    "severity": "high" if ca_score < 25 else "medium",
+                    "page": url, "metric": "content_answerability_score", "current_value": ca_score,
+                    "explanation": f"Content & answerability score is {ca_score}/100 for this {page_type} page - questions are not clearly answered, or no question-style headings/FAQ exist.",
+                    "recommended_fix": "Add direct, concise answers (40-300 chars) immediately after question-style headings, and consider an FAQ section.",
+                })
+            elif ca_score is not None and ca_score >= 70:
+                strengths.append({"page": url, "title": "Strong answerability", "detail": f"Content & answerability score {ca_score}/100."})
+
+        # --- Entity ---
+        if s.get("entity_score") is not None:
+            if s["entity_score"] < 40:
+                issues.append({
+                    "title": "Weak entity information",
+                    "severity": "medium", "page": url, "metric": "entity_score", "current_value": s["entity_score"],
+                    "explanation": f"Entity score is {s['entity_score']}/100 - the page doesn't clearly establish named entities (organizations, people, products) relevant to a '{page_type}' page.",
+                    "recommended_fix": "Mention your organization/brand and product names explicitly and consistently, and add Organization/Person schema.",
+                })
+            elif s["entity_score"] >= 70:
+                strengths.append({"page": url, "title": "Strong entity clarity", "detail": f"Entity score {s['entity_score']}/100."})
+
+        # --- Trust / author (only where relevant to page type) ---
+        if page_type in TYPES_WHERE_AUTHOR_RELEVANT:
+            if not m.get("author_entity_present") or (m.get("author_expertise") in (0, None)):
+                issues.append({
+                    "title": "No author expertise information detected",
+                    "severity": "medium", "page": url, "metric": "author_expertise", "current_value": m.get("author_expertise"),
+                    "explanation": f"No author byline or credentials were found on this {page_type} page, where authorship signals matter for trust.",
+                    "recommended_fix": "Add a visible author byline with credentials, or an author bio linking to their expertise.",
+                })
+
+        # --- Citation potential ---
+        if s.get("citation_potential_score") is not None and s["citation_potential_score"] < 40:
+            issues.append({
+                "title": "Low citation potential",
+                "severity": "low", "page": url, "metric": "citation_potential_score", "current_value": s["citation_potential_score"],
+                "explanation": f"Citation potential score is {s['citation_potential_score']}/100 - the page has few unique statistics, quotes, or attributed sources an AI system could cite.",
+                "recommended_fix": "Add original statistics, data points, or quotable expert statements with clear sourcing.",
+            })
+        elif s.get("citation_potential_score", 0) >= 70:
+            strengths.append({"page": url, "title": "Strong citation potential", "detail": f"Citation potential score {s['citation_potential_score']}/100."})
+
+        # --- Retrieval / JS dependency: only flagged when there's actual evidence content is JS-gated ---
+        rendering = m.get("content_rendering", {})
+        if rendering.get("retrieval_status") == "js_dependent":
+            issues.append({
+                "title": "Critical content is not present in server-rendered HTML",
+                "severity": "medium", "page": url, "metric": "retrieval_rendering",
+                "current_value": rendering.get("rendering_dependency_ratio"),
+                "explanation": f"Raw (non-JS) fetch returned only {rendering.get('raw_word_count', 0)} words vs {rendering.get('rendered_word_count', 0)} rendered - most content is injected by JavaScript, which some AI crawlers do not execute.",
+                "recommended_fix": "Ensure key content is present in server-rendered HTML (SSR) or a no-JS fallback.",
+            })
+        elif rendering.get("retrieval_status") == "good":
+            strengths.append({"page": url, "title": "Content is server-rendered", "detail": "Critical text content is present without executing JavaScript."})
+
+        # --- Content structure ---
+        if m.get("heading_structure_score", 0) < 50:
+            issues.append({
+                "title": "Poor content structure",
+                "severity": "low", "page": url, "metric": "heading_structure_score", "current_value": m.get("heading_structure_score", 0),
+                "explanation": f"Heading structure score is {m.get('heading_structure_score', 0)}/100.",
+                "recommended_fix": "Use a single H1 followed by a logical H2/H3 hierarchy so AI systems can segment the content.",
+            })
+
+        # --- Structured data: only an issue when a relevant schema type is actually missing ---
+        relevant_missing = []
+        if page_type in TYPES_WHERE_ARTICLE_SCHEMA_RELEVANT and not m.get("article_schema"):
+            relevant_missing.append("Article")
+        if page_type in TYPES_WHERE_PRODUCT_SCHEMA_RELEVANT and not m.get("product_schema"):
+            relevant_missing.append("Product")
+        if page_type in TYPES_WHERE_FAQ_RELEVANT and m.get("question_count", 0) >= 2 and not m.get("faq_schema"):
+            relevant_missing.append("FAQPage")
+        if relevant_missing:
+            issues.append({
+                "title": "Missing structured data",
+                "severity": "medium", "page": url, "metric": "schema_present", "current_value": m.get("schema_types"),
+                "explanation": f"No {'/'.join(relevant_missing)} schema was detected, though it is relevant for a '{page_type}' page.",
+                "recommended_fix": f"Add {'/'.join(relevant_missing)} JSON-LD structured data.",
+            })
+
+        # --- Freshness: unknown is reported separately, never as "outdated" ---
+        if m.get("freshness_status") == "unknown" and page_type in TYPES_WHERE_FRESHNESS_RELEVANT:
+            unknowns.append({"page": url, "metric": "freshness", "reason": "No publish/update date could be detected on this page, where freshness is typically relevant."})
+        elif m.get("freshness_status") in ("stale", "very_stale"):
+            issues.append({
+                "title": "Content appears stale",
+                "severity": "low" if m.get("freshness_status") == "stale" else "medium",
+                "page": url, "metric": "freshness_status", "current_value": m.get("content_age_days"),
+                "explanation": f"A reliable date was detected and the content is {m.get('content_age_days')} days old.",
+                "recommended_fix": "Review and update the content, then refresh the visible date / article:modified_time meta tag.",
+            })
+
+        for u in p.get("unknown_metrics", []):
+            unknowns.append({"page": url, **u})
+
+    severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+    issues.sort(key=lambda x: severity_order.get(x["severity"], 4))
+    return issues[:30], strengths[:20], unknowns[:30]
+
+
+# ==============================
+# TOP-LEVEL ENTRYPOINT
+# ==============================
+async def run_ai_visibility_analysis(base_url, max_pages=5, max_concurrent=1, use_ai=False, target_query=None):
+    """
+    Fetches pages (once) and computes AI-readiness metrics for each.
+
+    Returns a dict with an `ai_readiness_score` (transparent 0-100 proxy
+    score for how well pages are prepared to be understood/cited by AI
+    systems) plus per-page results, page-type classification, issues,
+    strengths, and unknown metrics. `actual_ai_visibility` is always
+    "not_measured" - this crawler has no access to real AI-query/citation
+    data, so it never fabricates one.
+    """
+    if not base_url:
+        raise ValueError("base_url is required")
+
+    domain = urlparse(base_url).netloc
+
+    urls = await discover_urls_parallel(base_url, max_pages)
+    if not urls:
+        urls = [base_url]
+
+    playwright_data = await fetch_all_pages_parallel(urls, max_concurrent)
+    if not playwright_data:
+        return {
+            "status": "error",
+            "message": "Failed to fetch any pages. Site may be blocking bots or require authentication.",
+        }
+
+    page_results = []
+    for seo_data in playwright_data:
+        result = await analyze_page_ai_visibility(seo_data, domain, use_ai=use_ai, target_query=target_query)
+        page_results.append(result)
+
+    valid_scores = [p["scores"]["ai_readiness_score"] for p in page_results if p.get("raw_metrics")]
+    overall_score = round(sum(valid_scores) / len(valid_scores)) if valid_scores else 0
+
+    def _avg(key):
+        vals = [p["scores"][key] for p in page_results if p.get("raw_metrics") and p["scores"].get(key) is not None]
+        return round(sum(vals) / len(vals)) if vals else None
+
+    category_averages = {
+        "semantic_score": _avg("semantic_score"),
+        "content_answerability_score": _avg("content_answerability_score"),
+        "entity_score": _avg("entity_score"),
+        "trust_score": _avg("trust_score"),
+        "citation_potential_score": _avg("citation_potential_score"),
+        "retrieval_score": _avg("retrieval_score"),
+        "structured_data_score": _avg("structured_data_score"),
+        "freshness_score": _avg("freshness_score"),
+    }
+
+    page_type_breakdown = dict(Counter(p.get("page_type", "unknown") for p in page_results))
+
+    issues, strengths, unknowns = _build_issues_and_strengths(page_results)
+
+    pages_summary = [
+        {
+            "url": p["url"],
+            "title": p["title"],
+            **p["ai_visibility"],
+        }
+        for p in page_results
+    ]
+
+    return {
+        "status": "success",
+        "url": base_url,
+        "pages_analyzed": len(page_results),
+        "page_type_breakdown": page_type_breakdown,
+        "target_query": target_query,
+        "ai_readiness_score": overall_score,
+        # legacy alias for existing frontend/API consumers
+        "ai_visibility_score": overall_score,
+        "actual_ai_visibility": {
+            "status": "not_measured",
+            "reason": "Real AI-query citation/mention data is not available to this crawler. This score reflects readiness proxies only, not observed visibility.",
+        },
+        "category_scores": category_averages,
+        "issues": issues,
+        "strengths": strengths,
+        "unknown_metrics": unknowns,
+        "results_preview": pages_summary,
+    }

app.py

@@ -0,0 +1,1479 @@
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import FileResponse, HTMLResponse
+from fastapi.middleware.cors import CORSMiddleware
+import os
+import asyncio
+import uuid
+import csv
+from datetime import datetime
+
+# Import your SEO analyzer functions
+from seo_analyzer import run_seo_analysis_fastapi
+from ai_visibility import run_ai_visibility_analysis
+
+app = FastAPI(
+    title="SEO Analysis API", 
+    version="2.0",
+    description="Advanced SEO analysis with AI-powered suggestions"
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Simple HTML UI - No static files needed
+HTML_TEMPLATE = """
+<!DOCTYPE html>
+<html>
+<head>
+    <title>SEO Analysis Tool</title>
+    <style>
+        * { margin: 0; padding: 0; box-sizing: border-box; }
+        body { font-family: Arial, sans-serif; background: #f5f5f5; padding: 20px; }
+        .container { max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 10px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }
+        h1 { color: #333; margin-bottom: 20px; text-align: center; }
+        .form-group { margin-bottom: 20px; }
+        label { display: block; margin-bottom: 5px; font-weight: bold; color: #555; }
+        input[type="text"], input[type="number"] { width: 100%; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-size: 16px; }
+        .checkbox-group { display: flex; align-items: center; gap: 10px; }
+        input[type="checkbox"] { width: 20px; height: 20px; }
+        button { background: #007bff; color: white; border: none; padding: 12px 30px; border-radius: 5px; cursor: pointer; font-size: 16px; margin-right: 10px; }
+        button:hover { background: #0056b3; }
+        button:disabled { background: #6c757d; cursor: not-allowed; }
+        .ai-visibility-btn { background: #6f42c1; }
+        .ai-visibility-btn:hover { background: #5a32a3; }
+        .loading { display: none; color: #007bff; margin: 20px 0; }
+        .results { margin-top: 30px; display: none; }
+        .ai-score-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 10px; margin: 15px 0; }
+        .ai-score-tile { background: #f3f0fa; border-radius: 6px; padding: 12px; text-align: center; }
+        .ai-score-tile .val { font-size: 24px; font-weight: bold; color: #6f42c1; }
+        .ai-score-tile .lbl { font-size: 12px; color: #555; margin-top: 4px; }
+        .ai-issue { background: white; border-left: 4px solid #6f42c1; margin: 10px 0; padding: 10px; border-radius: 5px; }
+        .score { font-size: 48px; font-weight: bold; text-align: center; margin: 20px 0; }
+        .score.good { color: #28a745; }
+        .score.average { color: #ffc107; }
+        .score.poor { color: #dc3545; }
+        .section { margin: 20px 0; padding: 20px; border-radius: 5px; }
+        .strengths { background: #d4edda; border-left: 4px solid #28a745; }
+        .faults { background: #f8d7da; border-left: 4px solid #dc3545; }
+        .suggestions { background: #fff3cd; border-left: 4px solid #ffc107; }
+        .suggestion-item { margin: 10px 0; padding: 10px; background: white; border-radius: 5px; }
+        .error { background: #f8d7da; color: #721c24; padding: 15px; border-radius: 5px; margin: 20px 0; }
+        .download-btn { background: #28a745; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; }
+        .download-btn:hover { background: #1e7e34; }
+        .preview { background: #e9ecef; padding: 15px; border-radius: 5px; margin: 10px 0; }
+        .api-test { background: #d1ecf1; padding: 15px; border-radius: 5px; margin: 10px 0; }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>🚀 SEO Analysis Tool</h1>
+        
+        <div class="form-group">
+            <label for="url">Website URL:</label>
+            <input type="text" id="url" placeholder="https://example.com" value="https://example.com">
+        </div>
+        
+        <div class="form-group">
+            <label for="maxPages">Max Pages to Analyze:</label>
+            <input type="number" id="maxPages" value="3" min="1" max="20">
+        </div>
+        
+        <div class="form-group">
+            <label for="maxConcurrent">Max Concurrent Browsers:</label>
+            <input type="number" id="maxConcurrent" value="1" min="1" max="5">
+        </div>
+        
+        <div class="form-group checkbox-group">
+            <input type="checkbox" id="useAI" checked>
+            <label for="useAI">Enable AI Suggestions</label>
+        </div>
+        
+        <button onclick="analyzeSEO()">Analyze SEO</button>
+        <button onclick="analyzeAIVisibility()" class="ai-visibility-btn"> AI Visibility Analysis</button>
+        <button onclick="testAPI()" style="background: #6c757d;">Test API</button>
+
+        <div id="loading" class="loading">
+            🔍 Analyzing website... This may take a few minutes.
+        </div>
+        <div id="aiLoading" class="loading">
+            Running AI Visibility analysis... This may take a few minutes.
+        </div>
+
+        <div id="results" class="results"></div>
+        <div id="aiResults" class="results"></div>
+    </div>
+    <script>
+        async function analyzeAIVisibility() {
+            const url = document.getElementById('url').value;
+            const maxPages = document.getElementById('maxPages').value;
+            const maxConcurrent = document.getElementById('maxConcurrent').value;
+            const useAI = document.getElementById('useAI').checked;
+
+            if (!url) {
+                alert('Please enter a website URL');
+                return;
+            }
+
+            const loading = document.getElementById('aiLoading');
+            const results = document.getElementById('aiResults');
+
+            loading.style.display = 'block';
+            results.style.display = 'none';
+            results.innerHTML = '';
+
+            try {
+                const apiUrl = `/analyze-ai-visibility?url=${encodeURIComponent(url)}&max_pages=${maxPages}&use_ai=${useAI}&max_concurrent=${maxConcurrent}`;
+                const response = await fetch(apiUrl);
+                const data = await response.json();
+                displayAIVisibilityResults(data);
+            } catch (error) {
+                results.innerHTML = `<div class="error">Error: ${error.message}</div>`;
+                results.style.display = 'block';
+            } finally {
+                loading.style.display = 'none';
+            }
+        }
+
+        function displayAIVisibilityResults(data) {
+            const results = document.getElementById('aiResults');
+            let html = '';
+
+            if (data.status === 'error') {
+                html = `<div class="error">${data.message}</div>`;
+                results.innerHTML = html;
+                results.style.display = 'block';
+                return;
+            }
+
+            const fmt = (v) => (v === null || v === undefined) ? 'Unknown' : v;
+
+            let scoreClass = 'poor';
+            if (data.ai_readiness_score >= 70) scoreClass = 'good';
+            else if (data.ai_readiness_score >= 50) scoreClass = 'average';
+
+            html += `
+                <div class="score ${scoreClass}">${data.ai_readiness_score}/100</div>
+                <p style="text-align: center; margin-bottom: 4px;">
+                    AI Readiness across <strong>${data.pages_analyzed}</strong> pages from <strong>${data.url}</strong>
+                </p>
+                <p style="text-align: center; color: #888; font-size: 0.85em; margin-bottom: 20px;">
+                    Actual AI Visibility (real citation/mention data): <em>${data.actual_ai_visibility ? data.actual_ai_visibility.status : 'not_measured'}</em> - this score reflects readiness proxies, not observed AI citations.
+                </p>
+            `;
+
+            if (data.page_type_breakdown) {
+                const types = Object.entries(data.page_type_breakdown).map(([t, n]) => `${t} (${n})`).join(', ');
+                html += `<p style="text-align:center; color:#555; margin-bottom: 15px;"><strong>Page types detected:</strong> ${types}</p>`;
+            }
+
+            if (data.category_scores) {
+                const labels = {
+                    semantic_score: 'Semantic',
+                    content_answerability_score: 'Content & Answerability',
+                    entity_score: 'Entity',
+                    trust_score: 'Trust',
+                    structured_data_score: 'Structured Data',
+                    retrieval_score: 'Retrieval',
+                    citation_potential_score: 'Citation Potential',
+                    freshness_score: 'Freshness'
+                };
+                html += `<div class="ai-score-grid">`;
+                for (const key in labels) {
+                    const val = fmt(data.category_scores[key]);
+                    html += `<div class="ai-score-tile"><div class="val">${val}</div><div class="lbl">${labels[key]}</div></div>`;
+                }
+                html += `</div>`;
+            }
+
+            if (data.strengths && data.strengths.length > 0) {
+                html += `<div class="section strengths"><h3>✅ Strengths (${data.strengths.length})</h3>`;
+                data.strengths.forEach(st => {
+                    html += `<div class="ai-issue" style="border-left-color:#28a745;"><strong>${st.title}</strong><br><span style="color:#666;">Page: ${st.page}</span><br><span>${st.detail || ''}</span></div>`;
+                });
+                html += `</div>`;
+            }
+
+            if (data.issues && data.issues.length > 0) {
+                html += `<div class="section suggestions"><h3>⚠️ AI Visibility Issues (${data.issues.length})</h3>`;
+                data.issues.forEach(issue => {
+                    html += `
+                        <div class="ai-issue">
+                            <strong>${issue.title}</strong> <span style="color: #6f42c1; font-size: 0.9em;">(Severity: ${issue.severity})</span><br>
+                            <span style="color: #666;">Page: ${issue.page}</span><br>
+                            <span>${issue.explanation}</span><br>
+                            <span style="color: #666; font-size: 0.9em;">💡 Fix: ${issue.recommended_fix}</span>
+                        </div>
+                    `;
+                });
+                html += `</div>`;
+            }
+
+            if (data.unknown_metrics && data.unknown_metrics.length > 0) {
+                html += `<div class="section" style="background:#e9ecef;"><h3>❔ Unknown (not enough evidence - not the same as poor)</h3>`;
+                data.unknown_metrics.forEach(u => {
+                    html += `<div class="ai-issue" style="border-left-color:#6c757d;"><strong>${u.metric}</strong><br><span style="color:#666;">Page: ${u.page}</span><br><span>${u.reason}</span></div>`;
+                });
+                html += `</div>`;
+            }
+
+            if (data.results_preview && data.results_preview.length > 0) {
+                html += `<div class="section"><h3>📄 Per-Page AI Visibility (${data.results_preview.length})</h3>`;
+                data.results_preview.forEach(page => {
+                    html += `
+                        <div class="preview">
+                            <p><strong>URL:</strong> ${page.url}</p>
+                            <p><strong>Page Type:</strong> ${page.page_type} (${Math.round((page.page_type_confidence || 0) * 100)}% confidence)</p>
+                            <p><strong>AI Readiness Score:</strong> ${page.ai_readiness_score}/100</p>
+                            <p><strong>Topic Clarity:</strong> ${page.topic_clarity}</p>
+                            <p><strong>Semantic Relevance:</strong> ${fmt(page.semantic_relevance)}</p>
+                            <p><strong>Content Completeness:</strong> ${page.content_completeness}</p>
+                            <p><strong>Entity Clarity:</strong> ${fmt(page.entity_clarity)}</p>
+                            <p><strong>Answer Coverage:</strong> ${fmt(page.answer_coverage)}</p>
+                            <p><strong>Schema Quality:</strong> ${page.schema_quality}</p>
+                            <p><strong>Crawlability:</strong> ${page.crawlability}</p>
+                            <p><strong>Content Structure:</strong> ${page.content_structure}</p>
+                            <p><strong>Citation Potential:</strong> ${page.citation_potential}</p>
+                            <p><strong>Freshness:</strong> ${page.freshness_status} ${fmt(page.freshness)}</p>
+                        </div>
+                    `;
+                });
+                html += `</div>`;
+            }
+
+            results.innerHTML = html;
+            results.style.display = 'block';
+        }
+
+        async function analyzeSEO() {
+            const url = document.getElementById('url').value;
+            const maxPages = document.getElementById('maxPages').value;
+            const maxConcurrent = document.getElementById('maxConcurrent').value;
+            const useAI = document.getElementById('useAI').checked;
+            
+            if (!url) {
+                alert('Please enter a website URL');
+                return;
+            }
+            
+            const loading = document.getElementById('loading');
+            const results = document.getElementById('results');
+            
+            loading.style.display = 'block';
+            results.style.display = 'none';
+            results.innerHTML = '';
+            
+            try {
+                const apiUrl = `/analyze?url=${encodeURIComponent(url)}&max_pages=${maxPages}&use_ai=${useAI}&max_concurrent=${maxConcurrent}`;
+                console.log('Calling API:', apiUrl);
+                
+                const response = await fetch(apiUrl);
+                const data = await response.json();
+                
+                displayResults(data);
+            } catch (error) {
+                results.innerHTML = `<div class="error">Error: ${error.message}</div>`;
+                results.style.display = 'block';
+            } finally {
+                loading.style.display = 'none';
+            }
+        }
+        
+        async function testAPI() {
+            const loading = document.getElementById('loading');
+            const results = document.getElementById('results');
+            
+            loading.style.display = 'block';
+            results.style.display = 'none';
+            
+            try {
+                const response = await fetch('/health');
+                const data = await response.json();
+                
+                results.innerHTML = `
+                    <div class="api-test">
+                        <h3>✅ API Health Check</h3>
+                        <p><strong>Status:</strong> ${data.status}</p>
+                        <p><strong>Message:</strong> ${data.message}</p>
+                        <p><strong>Timestamp:</strong> ${data.timestamp}</p>
+                    </div>
+                `;
+                results.style.display = 'block';
+            } catch (error) {
+                results.innerHTML = `<div class="error">API Test Failed: ${error.message}</div>`;
+                results.style.display = 'block';
+            } finally {
+                loading.style.display = 'none';
+            }
+        }
+        
+        function displayResults(data) {
+            const results = document.getElementById('results');
+            let html = '';
+            
+            if (data.status === 'error') {
+                html = `<div class="error">${data.message}</div>`;
+            } else {
+                // Score display
+                let scoreClass = 'poor';
+                if (data.overall_score >= 70) scoreClass = 'good';
+                else if (data.overall_score >= 50) scoreClass = 'average';
+                
+                html += `
+                    <div class="score ${scoreClass}">${data.overall_score}/100</div>
+                    <p style="text-align: center; margin-bottom: 20px;">
+                        Analyzed <strong>${data.pages_analyzed}</strong> pages from <strong>${data.url}</strong>
+                    </p>
+                `;
+                
+                // Download button
+                if (data.csv_download) {
+                    html += `
+                        <div style="text-align: center; margin: 20px 0;">
+                            <a href="${data.csv_download}" class="download-btn">
+                                📊 Download Full CSV Report
+                            </a>
+                        </div>
+                    `;
+                }
+                
+                // Strengths
+                if (data.strengths && data.strengths.length > 0) {
+                    html += `<div class="section strengths"><h3>✅ Strengths (${data.strengths.length})</h3>`;
+                    data.strengths.forEach(strength => {
+                        html += `<div class="suggestion-item"><strong>${strength.title}:</strong> ${strength.detail} <span style="color: #666; font-size: 0.9em;">(Impact: ${strength.impact})</span></div>`;
+                    });
+                    html += `</div>`;
+                }
+                
+                // Faults
+                if (data.faults && data.faults.length > 0) {
+                    html += `<div class="section faults"><h3>❌ Issues Found (${data.faults.length})</h3>`;
+                    data.faults.forEach(fault => {
+                        html += `<div class="suggestion-item"><strong>${fault.title}:</strong> ${fault.detail} <span style="color: #dc3545; font-weight: bold;">(Severity: ${fault.severity})</span>${fault.fix ? `<br><span style="color: #666; font-size: 0.9em;">💡 Fix: ${fault.fix}</span>` : ''}</div>`;
+                    });
+                    html += `</div>`;
+                }
+                
+                // AI Suggestions
+                if (data.suggestions && data.suggestions.length > 0) {
+                    html += `<div class="section suggestions"><h3>💡 AI Suggestions</h3>`;
+                    data.suggestions.forEach(suggestion => {
+                        html += `<div class="suggestion-item"><strong>${suggestion.title}:</strong> ${suggestion.detail}</div>`;
+                    });
+                    html += `</div>`;
+                }
+                
+                // Preview - SHOW ALL PAGES NOW with ALL DETAILED METRICS
+                if (data.results_preview && data.results_preview.length > 0) {
+                    html += `<div class="section"><h3>📄 All Analyzed Pages (${data.results_preview.length})</h3>`;
+                    data.results_preview.forEach((page, index) => {
+                        html += `
+                            <div class="preview">
+                                <p><strong>URL:</strong> ${page.url}</p>
+                                <p><strong>Score:</strong> ${page.seo_score}/100</p>
+                                <p><strong>Title:</strong> ${page.title || 'No title'}</p>
+                                <p><strong>Meta Description:</strong> ${page.meta_description || 'No meta description'}</p>
+                                <p><strong>Word Count:</strong> ${page.word_count}</p>
+                                <p><strong>H1 Count:</strong> ${page.h1_count}</p>
+                                <p><strong>H2 Count:</strong> ${page.h2_count}</p>
+                                <p><strong>H3 Count:</strong> ${page.h3_count || 0}</p>
+                                <p><strong>Heading Order:</strong> ${page.heading_order || 'None'}</p>
+                                <p><strong>Missing Alt Tags:</strong> ${page.missing_alt_tags || 0} of ${page.total_images || 0} images</p>
+                                <p><strong>Total Images:</strong> ${page.total_images || 0}</p>
+                                <p><strong>Small Images (&lt;100px):</strong> ${page.small_images || 0}</p>
+                                <p><strong>Large Images (&gt;2000px):</strong> ${page.large_images || 0}</p>
+                                <p><strong>Ideal Images (100-2000px):</strong> ${page.ideal_images || 0}</p>
+                                <p><strong>Internal Links:</strong> ${page.internal_links || 0}</p>
+                                <p><strong>External Links:</strong> ${page.external_links || 0}</p>
+                                <p><strong>Canonical Tag:</strong> ${page.canonical_tag ? '✅ Yes' : '❌ No'}</p>
+                                <p><strong>Robots Meta:</strong> ${page.robots_meta || 'None'}</p>
+                                <p><strong>Viewport:</strong> ${page.viewport_present ? '✅ Yes' : '❌ No'}</p>
+                                <p><strong>Schema Types:</strong> ${page.schema_types || 'No schema found'}</p>
+                                <p><strong>OpenGraph Tags:</strong> ${page.opengraph_tags || 0}</p>
+                                <p><strong>Twitter Tags:</strong> ${page.twitter_tags || 0}</p>
+                                <p><strong>Readability Score:</strong> ${page.readability_score || 0}/100</p>
+                                <p><strong>Grammar Errors:</strong> ${page.grammar_errors || 0}</p>
+                                <p><strong>Text/HTML Ratio:</strong> ${page.text_to_html_ratio || 0}%</p>
+                                <p><strong>Top Keywords:</strong> ${page.top_keywords || 'None'}</p>
+                                <p><strong>Load Time:</strong> ${page.load_time || 0}ms</p>
+                            </div>
+                        `;
+                    });
+                    html += `</div>`;
+                }
+            }
+            
+            results.innerHTML = html;
+            results.style.display = 'block';
+        }
+        
+        // Enter key support
+        document.getElementById('url').addEventListener('keypress', function(e) {
+            if (e.key === 'Enter') {
+                analyzeSEO();
+            }
+        });
+    </script>
+</body>
+</html>
+"""
+
+@app.get("/", response_class=HTMLResponse)
+async def root():
+    return HTML_TEMPLATE
+
+@app.get("/health")
+async def health_check():
+    return {
+        "status": "healthy", 
+        "message": "SEO Analysis API is running",
+        "timestamp": datetime.now().isoformat()
+    }
+
+@app.get("/analyze")
+async def analyze_seo(
+    url: str, 
+    max_pages: int = 5, 
+    use_ai: bool = True, 
+    max_concurrent: int = 1
+):
+    """
+    Analyze a single website for SEO optimization
+    
+    Args:
+        url: Website URL to analyze
+        max_pages: Maximum number of pages to analyze (default: 5)
+        use_ai: Enable AI-powered suggestions (default: True)
+        max_concurrent: Number of concurrent browsers (default: 1)
+    """
+    all_pages_summary = []
+    try:
+        print(f"🔍 Starting SEO analysis for: {url}")
+        
+        # Run SEO analysis
+        results, csv_path = await run_seo_analysis_fastapi(
+            base_url=url, 
+            max_pages=max_pages,
+            use_ai=use_ai,
+            max_concurrent=max_concurrent,
+            download=False
+        )
+        
+        # Calculate overall metrics
+        avg_score = 0
+        total_pages = len(results)
+        
+        if results and 'error' not in results[0]:
+            avg_score = round(sum(page.get("seo_score", 0) for page in results) / len(results), 1)
+        
+        # Extract strengths and faults
+        strengths = []
+        faults = []
+        suggestions = []
+        
+        if results and 'error' not in results[0]:
+            # Create complete page summaries for ALL pages with ALL metrics
+            # all_pages_summary = []
+            for page in results:
+                page_summary = {
+                    "url": page.get("url", ""),
+                    "title": page.get("title", ""),
+                    "meta_description": page.get("meta_description", ""),
+                    "h1_count": page.get("h1_count", 0),
+                    "h2_count": page.get("h2_count", 0),
+                    "h3_count": page.get("h3_count", 0),
+                    "heading_order": page.get("heading_order", ""),
+                    "missing_alt_tags": page.get("missing_alt_tags", 0),
+                    "total_images": page.get("total_images", 0),
+                    "small_images": page.get("small_images", 0),
+                    "large_images": page.get("large_images", 0),
+                    "ideal_images": page.get("ideal_images", 0),
+                    "internal_links": page.get("internal_links", 0),
+                    "external_links": page.get("external_links", 0),
+                    "canonical_tag": page.get("canonical_tag", False),
+                    "robots_meta": page.get("robots_meta", ""),
+                    "viewport_present": page.get("viewport_present", False),
+                    "schema_types": page.get("schema_types", ""),
+                    "opengraph_tags": page.get("opengraph_tags", 0),
+                    "twitter_tags": page.get("twitter_tags", 0),
+                    "word_count": page.get("word_count", 0),
+                    "readability_score": page.get("readability_score", 0),
+                    "grammar_errors": page.get("grammar_errors", 0),
+                    "text_to_html_ratio": page.get("text_to_html_ratio", 0),
+                    "top_keywords": page.get("top_keywords", ""),
+                    "load_time": page.get("load_time", 0),
+                    "seo_score": page.get("seo_score", 0)
+                }
+                all_pages_summary.append(page_summary)
+            
+            # Sort pages by score (highest first)
+            all_pages_summary.sort(key=lambda x: x["seo_score"], reverse=True)
+            
+            # Calculate comprehensive metrics
+            total_pages = len(results)
+            
+            # 1. TITLE ANALYSIS
+            title_lengths = [len(page.get('title', '')) for page in results if page.get('title')]
+            good_title_pages = sum(1 for page in results if page.get('title') and 45 <= len(page.get('title', '')) <= 65)
+            missing_title_pages = sum(1 for page in results if not page.get('title') or page.get('title', '').strip() == '')
+            short_title_pages = sum(1 for page in results if page.get('title') and len(page.get('title', '')) < 45)
+            long_title_pages = sum(1 for page in results if page.get('title') and len(page.get('title', '')) > 65)
+            
+            # 2. META DESCRIPTION ANALYSIS
+            meta_lengths = [len(page.get('meta_description', '')) for page in results if page.get('meta_description')]
+            good_meta_pages = sum(1 for page in results if page.get('meta_description') and 120 <= len(page.get('meta_description', '')) <= 155)
+            missing_meta_pages = sum(1 for page in results if not page.get('meta_description') or page.get('meta_description', '').strip() == '')
+            short_meta_pages = sum(1 for page in results if page.get('meta_description') and len(page.get('meta_description', '')) < 120)
+            long_meta_pages = sum(1 for page in results if page.get('meta_description') and len(page.get('meta_description', '')) > 155)
+            
+            # 3. HEADING STRUCTURE ANALYSIS
+            proper_h1_pages = sum(1 for page in results if page.get('h1_count', 0) == 1)
+            multiple_h1_pages = sum(1 for page in results if page.get('h1_count', 0) > 1)
+            missing_h1_pages = sum(1 for page in results if page.get('h1_count', 0) == 0)
+            good_heading_hierarchy_pages = sum(1 for page in results if page.get('heading_order') and 'H1 → H2 → H3' in page.get('heading_order', ''))
+            has_h2_pages = sum(1 for page in results if page.get('h2_count', 0) > 0)
+            has_h3_pages = sum(1 for page in results if page.get('h3_count', 0) > 0)
+            
+            # 4. IMAGE ANALYSIS
+            total_images_all = sum(page.get('total_images', 0) for page in results)
+            total_missing_alt = sum(page.get('missing_alt_tags', 0) for page in results)
+            total_ideal_images = sum(page.get('ideal_images', 0) for page in results)
+            total_small_images = sum(page.get('small_images', 0) for page in results)
+            total_large_images = sum(page.get('large_images', 0) for page in results)
+            pages_with_images = sum(1 for page in results if page.get('total_images', 0) > 0)
+            pages_without_images = sum(1 for page in results if page.get('total_images', 0) == 0)
+            perfect_image_pages = sum(1 for page in results if page.get('missing_alt_tags', 0) == 0 and page.get('total_images', 0) > 0)
+            
+            # 5. CONTENT ANALYSIS
+            good_wordcount_pages = sum(1 for page in results if page.get('word_count', 0) >= 500)
+            thin_content_pages = sum(1 for page in results if page.get('word_count', 0) < 300)
+            excellent_readability_pages = sum(1 for page in results if page.get('readability_score', 0) >= 70)
+            poor_readability_pages = sum(1 for page in results if page.get('readability_score', 0) < 50)
+            grammar_error_pages = sum(1 for page in results if page.get('grammar_errors', 0) > 0)
+            good_text_ratio_pages = sum(1 for page in results if page.get('text_to_html_ratio', 0) >= 25)
+            low_text_ratio_pages = sum(1 for page in results if page.get('text_to_html_ratio', 0) < 15)
+            
+            # 6. LINK ANALYSIS
+            internal_links_total = sum(page.get('internal_links', 0) for page in results)
+            external_links_total = sum(page.get('external_links', 0) for page in results)
+            good_internal_link_pages = sum(1 for page in results if page.get('internal_links', 0) >= 5)
+            good_external_link_pages = sum(1 for page in results if page.get('external_links', 0) >= 2 and page.get('external_links', 0) <= 10)
+            no_internal_links_pages = sum(1 for page in results if page.get('internal_links', 0) == 0)
+            excessive_external_links_pages = sum(1 for page in results if page.get('external_links', 0) > 15)
+            
+            # 7. TECHNICAL SEO ANALYSIS
+            mobile_friendly_pages = sum(1 for page in results if page.get('viewport_present', False))
+            canonical_pages = sum(1 for page in results if page.get('canonical_tag', False))
+            has_schema_pages = sum(1 for page in results if page.get('schema_types') and page.get('schema_types') != "No schema found" and page.get('schema_types', '').strip() != '')
+            good_opengraph_pages = sum(1 for page in results if page.get('opengraph_tags', 0) >= 5)
+            good_twitter_pages = sum(1 for page in results if page.get('twitter_tags', 0) >= 5)
+            noindex_pages = sum(1 for page in results if page.get('robots_meta') and 'noindex' in str(page.get('robots_meta', '')).lower())
+            
+            # 8. PERFORMANCE ANALYSIS
+            fast_load_pages = sum(1 for page in results if page.get('load_time', 0) < 2000)
+            slow_load_pages = sum(1 for page in results if page.get('load_time', 0) > 4000)
+            
+            # 9. KEYWORD ANALYSIS
+            has_keywords_pages = sum(1 for page in results if page.get('top_keywords') and page.get('top_keywords', '').strip() != '')
+            
+            # STRENGTHS ANALYSIS
+            strengths = []
+            
+            if good_title_pages > 0:
+                strengths.append({
+                    "title": "Optimized Title Tags",
+                    "detail": f"{good_title_pages}/{total_pages} pages ({good_title_pages/total_pages*100:.0f}%) have well-optimized title tags (45-65 chars)",
+                    "impact": "high",
+                    "metric": f"{good_title_pages}/{total_pages} pages"
+                })
+            
+            if good_meta_pages > 0:
+                strengths.append({
+                    "title": "Proper Meta Descriptions", 
+                    "detail": f"{good_meta_pages}/{total_pages} pages ({good_meta_pages/total_pages*100:.0f}%) have well-formatted meta descriptions",
+                    "impact": "high",
+                    "metric": f"{good_meta_pages}/{total_pages} pages"
+                })
+            
+            if mobile_friendly_pages > 0:
+                strengths.append({
+                    "title": "Mobile Responsive",
+                    "detail": f"{mobile_friendly_pages}/{total_pages} pages ({mobile_friendly_pages/total_pages*100:.0f}%) are mobile-friendly", 
+                    "impact": "high",
+                    "metric": f"{mobile_friendly_pages}/{total_pages} pages"
+                })
+                
+            if proper_h1_pages > 0:
+                strengths.append({
+                    "title": "Proper H1 Structure",
+                    "detail": f"{proper_h1_pages}/{total_pages} pages ({proper_h1_pages/total_pages*100:.0f}%) have exactly one H1 heading",
+                    "impact": "high",
+                    "metric": f"{proper_h1_pages}/{total_pages} pages"
+                })
+            
+            if good_heading_hierarchy_pages > 0:
+                strengths.append({
+                    "title": "Good Heading Hierarchy",
+                    "detail": f"{good_heading_hierarchy_pages}/{total_pages} pages follow proper H1 → H2 → H3 structure",
+                    "impact": "medium",
+                    "metric": f"{good_heading_hierarchy_pages}/{total_pages} pages"
+                })
+            
+            if perfect_image_pages > 0 and pages_with_images > 0:
+                strengths.append({
+                    "title": "Image Optimization",
+                    "detail": f"{perfect_image_pages}/{pages_with_images} pages ({perfect_image_pages/pages_with_images*100:.0f}%) with images have perfect alt text",
+                    "impact": "medium",
+                    "metric": f"{perfect_image_pages}/{pages_with_images} pages"
+                })
+            
+            if good_wordcount_pages > 0:
+                strengths.append({
+                    "title": "Quality Content",
+                    "detail": f"{good_wordcount_pages}/{total_pages} pages ({good_wordcount_pages/total_pages*100:.0f}%) have comprehensive content (500+ words)",
+                    "impact": "high",
+                    "metric": f"{good_wordcount_pages}/{total_pages} pages"
+                })
+            
+            if excellent_readability_pages > 0:
+                strengths.append({
+                    "title": "Excellent Readability",
+                    "detail": f"{excellent_readability_pages}/{total_pages} pages ({excellent_readability_pages/total_pages*100:.0f}%) have high readability scores (70+)",
+                    "impact": "medium",
+                    "metric": f"{excellent_readability_pages}/{total_pages} pages"
+                })
+            
+            if canonical_pages > 0:
+                strengths.append({
+                    "title": "Canonical Tags",
+                    "detail": f"{canonical_pages}/{total_pages} pages ({canonical_pages/total_pages*100:.0f}%) have canonical tags preventing duplicate content",
+                    "impact": "high",
+                    "metric": f"{canonical_pages}/{total_pages} pages"
+                })
+            
+            if has_schema_pages > 0:
+                strengths.append({
+                    "title": "Schema Markup",
+                    "detail": f"{has_schema_pages}/{total_pages} pages ({has_schema_pages/total_pages*100:.0f}%) use structured data markup",
+                    "impact": "medium",
+                    "metric": f"{has_schema_pages}/{total_pages} pages"
+                })
+            
+            if good_internal_link_pages > 0:
+                strengths.append({
+                    "title": "Internal Linking",
+                    "detail": f"{good_internal_link_pages}/{total_pages} pages ({good_internal_link_pages/total_pages*100:.0f}%) have strong internal linking (5+ internal links)",
+                    "impact": "medium",
+                    "metric": f"{good_internal_link_pages}/{total_pages} pages"
+                })
+            
+            if fast_load_pages > 0:
+                strengths.append({
+                    "title": "Fast Loading Pages",
+                    "detail": f"{fast_load_pages}/{total_pages} pages ({fast_load_pages/total_pages*100:.0f}%) load quickly (<2 seconds)",
+                    "impact": "high",
+                    "metric": f"{fast_load_pages}/{total_pages} pages"
+                })
+            
+            if has_keywords_pages > 0:
+                strengths.append({
+                    "title": "Keyword Optimization",
+                    "detail": f"{has_keywords_pages}/{total_pages} pages ({has_keywords_pages/total_pages*100:.0f}%) show clear keyword targeting",
+                    "impact": "high",
+                    "metric": f"{has_keywords_pages}/{total_pages} pages"
+                })
+            
+            if good_opengraph_pages > 0:
+                strengths.append({
+                    "title": "Social Media Ready",
+                    "detail": f"{good_opengraph_pages}/{total_pages} pages ({good_opengraph_pages/total_pages*100:.0f}%) have comprehensive OpenGraph tags",
+                    "impact": "low",
+                    "metric": f"{good_opengraph_pages}/{total_pages} pages"
+                })
+            
+            # FAULTS ANALYSIS
+            faults = []
+            
+            if missing_title_pages > 0:
+                faults.append({
+                    "title": "Missing Title Tags",
+                    "detail": f"{missing_title_pages}/{total_pages} pages ({missing_title_pages/total_pages*100:.0f}%) are missing title tags",
+                    "severity": "critical",
+                    "pages": missing_title_pages,
+                    "fix": "Add descriptive title tags to every page"
+                })
+            
+            if missing_h1_pages > 0:
+                faults.append({
+                    "title": "Missing H1 Headings",
+                    "detail": f"{missing_h1_pages}/{total_pages} pages ({missing_h1_pages/total_pages*100:.0f}%) are missing H1 headings",
+                    "severity": "critical",
+                    "pages": missing_h1_pages,
+                    "fix": "Add a single, descriptive H1 heading to each page"
+                })
+            
+            if multiple_h1_pages > 0:
+                faults.append({
+                    "title": "Multiple H1 Headings",
+                    "detail": f"{multiple_h1_pages}/{total_pages} pages ({multiple_h1_pages/total_pages*100:.0f}%) have multiple H1 headings",
+                    "severity": "high",
+                    "pages": multiple_h1_pages,
+                    "fix": "Reduce to one H1 per page for better SEO"
+                })
+            
+            if missing_meta_pages > 0:
+                faults.append({
+                    "title": "Missing Meta Descriptions", 
+                    "detail": f"{missing_meta_pages}/{total_pages} pages ({missing_meta_pages/total_pages*100:.0f}%) are missing meta descriptions",
+                    "severity": "high",
+                    "pages": missing_meta_pages,
+                    "fix": "Add compelling meta descriptions to improve click-through rates"
+                })
+            
+            if total_missing_alt > 0 and total_images_all > 0:
+                alt_ratio = (total_missing_alt / total_images_all) * 100
+                faults.append({
+                    "title": "Missing Image Alt Text",
+                    "detail": f"{total_missing_alt}/{total_images_all} images ({alt_ratio:.0f}%) are missing alt text",
+                    "severity": "high" if alt_ratio > 20 else "medium",
+                    "pages": pages_with_images,
+                    "fix": "Add descriptive alt text to all images"
+                })
+            
+            if total_small_images > 0:
+                faults.append({
+                    "title": "Small Images Detected",
+                    "detail": f"{total_small_images} images are too small (<100px) which may appear blurry on high-resolution screens",
+                    "severity": "low",
+                    "pages": sum(1 for page in results if page.get('small_images', 0) > 0),
+                    "fix": "Use higher resolution images (minimum 100px width/height)"
+                })
+            
+            if total_large_images > 0:
+                faults.append({
+                    "title": "Oversized Images",
+                    "detail": f"{total_large_images} images are too large (>2000px) causing slow page loads",
+                    "severity": "medium",
+                    "pages": sum(1 for page in results if page.get('large_images', 0) > 0),
+                    "fix": "Optimize and resize large images to under 2000px"
+                })
+            
+            if thin_content_pages > 0:
+                faults.append({
+                    "title": "Thin Content",
+                    "detail": f"{thin_content_pages}/{total_pages} pages ({thin_content_pages/total_pages*100:.0f}%) have thin content (<300 words)",
+                    "severity": "high",
+                    "pages": thin_content_pages,
+                    "fix": "Expand content to provide more value (aim for 500+ words)"
+                })
+            
+            if poor_readability_pages > 0:
+                faults.append({
+                    "title": "Poor Readability",
+                    "detail": f"{poor_readability_pages}/{total_pages} pages ({poor_readability_pages/total_pages*100:.0f}%) have low readability scores (<50)",
+                    "severity": "medium",
+                    "pages": poor_readability_pages,
+                    "fix": "Simplify language, use shorter sentences and paragraphs"
+                })
+            
+            if grammar_error_pages > 0:
+                faults.append({
+                    "title": "Grammar Errors",
+                    "detail": f"{grammar_error_pages}/{total_pages} pages ({grammar_error_pages/total_pages*100:.0f}%) contain grammar errors",
+                    "severity": "medium",
+                    "pages": grammar_error_pages,
+                    "fix": "Review and correct grammar mistakes for better credibility"
+                })
+            
+            if low_text_ratio_pages > 0:
+                faults.append({
+                    "title": "Low Text to HTML Ratio",
+                    "detail": f"{low_text_ratio_pages}/{total_pages} pages ({low_text_ratio_pages/total_pages*100:.0f}%) have low text content (<15% text ratio)",
+                    "severity": "medium",
+                    "pages": low_text_ratio_pages,
+                    "fix": "Increase meaningful text content relative to HTML code"
+                })
+            
+            if no_internal_links_pages > 0:
+                faults.append({
+                    "title": "No Internal Links",
+                    "detail": f"{no_internal_links_pages}/{total_pages} pages ({no_internal_links_pages/total_pages*100:.0f}%) have no internal links",
+                    "severity": "high",
+                    "pages": no_internal_links_pages,
+                    "fix": "Add internal links to improve site structure and crawlability"
+                })
+            
+            if excessive_external_links_pages > 0:
+                faults.append({
+                    "title": "Excessive External Links",
+                    "detail": f"{excessive_external_links_pages}/{total_pages} pages have too many external links (>15)",
+                    "severity": "low",
+                    "pages": excessive_external_links_pages,
+                    "fix": "Reduce external links to maintain link equity"
+                })
+            
+            if mobile_friendly_pages < total_pages:
+                non_mobile_pages = total_pages - mobile_friendly_pages
+                faults.append({
+                    "title": "Non-Mobile Friendly Pages",
+                    "detail": f"{non_mobile_pages}/{total_pages} pages ({non_mobile_pages/total_pages*100:.0f}%) lack mobile viewport tag",
+                    "severity": "critical",
+                    "pages": non_mobile_pages,
+                    "fix": "Add viewport meta tag to all pages: <meta name='viewport' content='width=device-width, initial-scale=1'>"
+                })
+            
+            if noindex_pages > 0:
+                faults.append({
+                    "title": "Noindex Pages",
+                    "detail": f"{noindex_pages} pages are marked with 'noindex' and won't appear in search results",
+                    "severity": "critical",
+                    "pages": noindex_pages,
+                    "fix": "Remove 'noindex' from pages you want to rank unless intentionally hidden"
+                })
+            
+            if has_schema_pages < total_pages:
+                missing_schema_pages = total_pages - has_schema_pages
+                faults.append({
+                    "title": "Missing Schema Markup",
+                    "detail": f"{missing_schema_pages}/{total_pages} pages ({missing_schema_pages/total_pages*100:.0f}%) lack structured data markup",
+                    "severity": "medium",
+                    "pages": missing_schema_pages,
+                    "fix": "Add appropriate schema.org markup to improve rich snippets"
+                })
+            
+            if slow_load_pages > 0:
+                faults.append({
+                    "title": "Slow Loading Pages",
+                    "detail": f"{slow_load_pages}/{total_pages} pages ({slow_load_pages/total_pages*100:.0f}%) load slowly (>4 seconds)",
+                    "severity": "high",
+                    "pages": slow_load_pages,
+                    "fix": "Optimize images, minify CSS/JS, enable caching"
+                })
+            
+            if pages_without_images > 0:
+                faults.append({
+                    "title": "Pages Without Images",
+                    "detail": f"{pages_without_images}/{total_pages} pages ({pages_without_images/total_pages*100:.0f}%) have no images",
+                    "severity": "low",
+                    "pages": pages_without_images,
+                    "fix": "Add relevant images to improve engagement and visual appeal"
+                })
+            
+            # Limit to top 10 faults by severity
+            severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+            faults.sort(key=lambda x: severity_order[x["severity"]])
+            faults = faults[:15]
+
+        # Extract AI suggestions
+        if results and 'error' not in results[0] and use_ai:
+            for page in results:
+                if page.get('ai_suggestions') and page['ai_suggestions'] not in [
+                    "AI suggestions disabled - set valid OPENAI_API_KEY", 
+                    "AI disabled - set valid OPENAI_API_KEY",
+                    "AI suggestions disabled - set use_ai=True and OPENAI_API_KEY"
+                ]:
+                    # Extract key suggestions from AI output
+                    ai_text = page['ai_suggestions']
+                    
+                    # Take first few lines as suggestions
+                    lines = [line.strip() for line in ai_text.split('\n') if line.strip() and len(line.strip()) > 20]
+                    for line in lines[:3]:
+                        suggestions.append({
+                            "title": "SEO Recommendation",
+                            "detail": line[:150],
+                            "priority": "high"
+                        })
+
+        # Limit suggestions
+        suggestions = suggestions[:5]
+
+        return {
+            "status": "success",
+            "url": url,
+            "pages_analyzed": total_pages,
+            "overall_score": avg_score,
+            "strengths": strengths,
+            "faults": faults, 
+            "suggestions": suggestions,
+            # Return ALL pages summary with ALL metrics
+            "results_preview": all_pages_summary,
+            "csv_download": f"/download/{os.path.basename(csv_path)}" if results and 'error' not in results[0] else None
+        }
+        
+    except Exception as e:
+        print(f"❌ Analysis error: {e}")
+        return {
+            "status": "error",
+            "message": str(e)
+        }
+
+@app.get("/analyze-ai-visibility")
+async def analyze_ai_visibility(
+    url: str,
+    max_pages: int = 5,
+    use_ai: bool = True,
+    max_concurrent: int = 1,
+    target_query: str = None,
+):
+    """
+    Analyze a website's AI Visibility / AI Search Readiness.
+
+    Runs independently of /analyze - it fetches pages itself (once) and
+    computes proxy signals for how well content can be understood, retrieved,
+    cited and summarized by AI-powered search systems.
+
+    target_query (optional): a keyword/query to measure semantic_relevance
+    and search_intent_match against. Without it, those two metrics are
+    reported as unknown rather than guessed at.
+    """
+    try:
+        print(f"🤖 Starting AI Visibility analysis for: {url}")
+        result = await run_ai_visibility_analysis(
+            base_url=url,
+            max_pages=max_pages,
+            max_concurrent=max_concurrent,
+            use_ai=use_ai,
+            target_query=target_query,
+        )
+        return result
+    except Exception as e:
+        print(f"❌ AI Visibility analysis error: {e}")
+        return {
+            "status": "error",
+            "message": str(e)
+        }
+
+@app.get("/download/{filename}")
+async def download_file(filename: str):
+    """Download SEO analysis CSV report"""
+    file_path = f"/tmp/{filename}"
+    if os.path.exists(file_path):
+        return FileResponse(
+            file_path,
+            media_type='text/csv',
+            filename=f"seo_report.csv"
+        )
+    raise HTTPException(status_code=404, detail="File not found")
+
+if __name__ == "__main__":
+    import uvicorn
+    print("🚀 Starting SEO Analysis API...")
+    uvicorn.run(app, host="0.0.0.0", port=7860)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# from fastapi import FastAPI, HTTPException
+# from fastapi.responses import FileResponse, HTMLResponse
+# from fastapi.middleware.cors import CORSMiddleware
+# import os
+# import asyncio
+# import uuid
+# import csv
+# from datetime import datetime
+
+# # Import your SEO analyzer functions
+# from seo_analyzer import run_seo_analysis_fastapi
+
+# app = FastAPI(
+#     title="SEO Analysis API", 
+#     version="2.0",
+#     description="Advanced SEO analysis with AI-powered suggestions"
+# )
+
+# app.add_middleware(
+#     CORSMiddleware,
+#     allow_origins=["*"],
+#     allow_credentials=True,
+#     allow_methods=["*"],
+#     allow_headers=["*"],
+# )
+
+# # Simple HTML UI - No static files needed
+# HTML_TEMPLATE = """
+# <!DOCTYPE html>
+# <html>
+# <head>
+#     <title>SEO Analysis Tool</title>
+#     <style>
+#         * { margin: 0; padding: 0; box-sizing: border-box; }
+#         body { font-family: Arial, sans-serif; background: #f5f5f5; padding: 20px; }
+#         .container { max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 10px; box-shadow: 0 2px 10px rgba(0,0,0,0.1); }
+#         h1 { color: #333; margin-bottom: 20px; text-align: center; }
+#         .form-group { margin-bottom: 20px; }
+#         label { display: block; margin-bottom: 5px; font-weight: bold; color: #555; }
+#         input[type="text"], input[type="number"] { width: 100%; padding: 10px; border: 1px solid #ddd; border-radius: 5px; font-size: 16px; }
+#         .checkbox-group { display: flex; align-items: center; gap: 10px; }
+#         input[type="checkbox"] { width: 20px; height: 20px; }
+#         button { background: #007bff; color: white; border: none; padding: 12px 30px; border-radius: 5px; cursor: pointer; font-size: 16px; margin-right: 10px; }
+#         button:hover { background: #0056b3; }
+#         button:disabled { background: #6c757d; cursor: not-allowed; }
+#         .loading { display: none; color: #007bff; margin: 20px 0; }
+#         .results { margin-top: 30px; display: none; }
+#         .score { font-size: 48px; font-weight: bold; text-align: center; margin: 20px 0; }
+#         .score.good { color: #28a745; }
+#         .score.average { color: #ffc107; }
+#         .score.poor { color: #dc3545; }
+#         .section { margin: 20px 0; padding: 20px; border-radius: 5px; }
+#         .strengths { background: #d4edda; border-left: 4px solid #28a745; }
+#         .faults { background: #f8d7da; border-left: 4px solid #dc3545; }
+#         .suggestions { background: #fff3cd; border-left: 4px solid #ffc107; }
+#         .suggestion-item { margin: 10px 0; padding: 10px; background: white; border-radius: 5px; }
+#         .error { background: #f8d7da; color: #721c24; padding: 15px; border-radius: 5px; margin: 20px 0; }
+#         .download-btn { background: #28a745; color: white; text-decoration: none; padding: 10px 20px; border-radius: 5px; display: inline-block; }
+#         .download-btn:hover { background: #1e7e34; }
+#         .preview { background: #e9ecef; padding: 15px; border-radius: 5px; margin: 10px 0; }
+#         .api-test { background: #d1ecf1; padding: 15px; border-radius: 5px; margin: 10px 0; }
+#     </style>
+# </head>
+# <body>
+#     <div class="container">
+#         <h1>🚀 SEO Analysis Tool</h1>
+        
+#         <div class="form-group">
+#             <label for="url">Website URL:</label>
+#             <input type="text" id="url" placeholder="https://example.com" value="https://example.com">
+#         </div>
+        
+#         <div class="form-group">
+#             <label for="maxPages">Max Pages to Analyze:</label>
+#             <input type="number" id="maxPages" value="3" min="1" max="20">
+#         </div>
+        
+#         <div class="form-group">
+#             <label for="maxConcurrent">Max Concurrent Browsers:</label>
+#             <input type="number" id="maxConcurrent" value="1" min="1" max="5">
+#         </div>
+        
+#         <div class="form-group checkbox-group">
+#             <input type="checkbox" id="useAI" checked>
+#             <label for="useAI">Enable AI Suggestions</label>
+#         </div>
+        
+#         <button onclick="analyzeSEO()">Analyze SEO</button>
+#         <button onclick="testAPI()" style="background: #6c757d;">Test API</button>
+        
+#         <div id="loading" class="loading">
+#             🔍 Analyzing website... This may take a few minutes.
+#         </div>
+        
+#         <div id="results" class="results"></div>
+#     </div>
+#     <script>
+#         async function analyzeSEO() {
+#             const url = document.getElementById('url').value;
+#             const maxPages = document.getElementById('maxPages').value;
+#             const maxConcurrent = document.getElementById('maxConcurrent').value;
+#             const useAI = document.getElementById('useAI').checked;
+            
+#             if (!url) {
+#                 alert('Please enter a website URL');
+#                 return;
+#             }
+            
+#             const loading = document.getElementById('loading');
+#             const results = document.getElementById('results');
+            
+#             loading.style.display = 'block';
+#             results.style.display = 'none';
+#             results.innerHTML = '';
+            
+#             try {
+#                 const apiUrl = `/analyze?url=${encodeURIComponent(url)}&max_pages=${maxPages}&use_ai=${useAI}&max_concurrent=${maxConcurrent}`;
+#                 console.log('Calling API:', apiUrl);
+                
+#                 const response = await fetch(apiUrl);
+#                 const data = await response.json();
+                
+#                 displayResults(data);
+#             } catch (error) {
+#                 results.innerHTML = `<div class="error">Error: ${error.message}</div>`;
+#                 results.style.display = 'block';
+#             } finally {
+#                 loading.style.display = 'none';
+#             }
+#         }
+        
+#         async function testAPI() {
+#             const loading = document.getElementById('loading');
+#             const results = document.getElementById('results');
+            
+#             loading.style.display = 'block';
+#             results.style.display = 'none';
+            
+#             try {
+#                 const response = await fetch('/health');
+#                 const data = await response.json();
+                
+#                 results.innerHTML = `
+#                     <div class="api-test">
+#                         <h3>✅ API Health Check</h3>
+#                         <p><strong>Status:</strong> ${data.status}</p>
+#                         <p><strong>Message:</strong> ${data.message}</p>
+#                         <p><strong>Timestamp:</strong> ${data.timestamp}</p>
+#                     </div>
+#                 `;
+#                 results.style.display = 'block';
+#             } catch (error) {
+#                 results.innerHTML = `<div class="error">API Test Failed: ${error.message}</div>`;
+#                 results.style.display = 'block';
+#             } finally {
+#                 loading.style.display = 'none';
+#             }
+#         }
+        
+#         function displayResults(data) {
+#             const results = document.getElementById('results');
+#             let html = '';
+            
+#             if (data.status === 'error') {
+#                 html = `<div class="error">${data.message}</div>`;
+#             } else {
+#                 // Score display
+#                 let scoreClass = 'poor';
+#                 if (data.overall_score >= 70) scoreClass = 'good';
+#                 else if (data.overall_score >= 50) scoreClass = 'average';
+                
+#                 html += `
+#                     <div class="score ${scoreClass}">${data.overall_score}/100</div>
+#                     <p style="text-align: center; margin-bottom: 20px;">
+#                         Analyzed <strong>${data.pages_analyzed}</strong> pages from <strong>${data.url}</strong>
+#                     </p>
+#                 `;
+                
+#                 // Download button
+#                 if (data.csv_download) {
+#                     html += `
+#                         <div style="text-align: center; margin: 20px 0;">
+#                             <a href="${data.csv_download}" class="download-btn">
+#                                 📊 Download Full CSV Report
+#                             </a>
+#                         </div>
+#                     `;
+#                 }
+                
+#                 // Strengths
+#                 if (data.strengths && data.strengths.length > 0) {
+#                     html += `<div class="section strengths"><h3>✅ Strengths</h3>`;
+#                     data.strengths.forEach(strength => {
+#                         html += `<div class="suggestion-item"><strong>${strength.title}:</strong> ${strength.detail}</div>`;
+#                     });
+#                     html += `</div>`;
+#                 }
+                
+#                 // Faults
+#                 if (data.faults && data.faults.length > 0) {
+#                     html += `<div class="section faults"><h3>❌ Issues Found</h3>`;
+#                     data.faults.forEach(fault => {
+#                         html += `<div class="suggestion-item"><strong>${fault.title}:</strong> ${fault.detail} (Severity: ${fault.severity})</div>`;
+#                     });
+#                     html += `</div>`;
+#                 }
+                
+#                 // AI Suggestions
+#                 if (data.suggestions && data.suggestions.length > 0) {
+#                     html += `<div class="section suggestions"><h3>💡 AI Suggestions</h3>`;
+#                     data.suggestions.forEach(suggestion => {
+#                         html += `<div class="suggestion-item"><strong>${suggestion.title}:</strong> ${suggestion.detail}</div>`;
+#                     });
+#                     html += `</div>`;
+#                 }
+                
+#                 // Preview - SHOW ALL PAGES NOW with ALL DETAILED METRICS
+#                 if (data.results_preview && data.results_preview.length > 0) {
+#                     html += `<div class="section"><h3>📄 All Analyzed Pages (${data.results_preview.length})</h3>`;
+#                     data.results_preview.forEach((page, index) => {
+#                         html += `
+#                             <div class="preview">
+#                                 <p><strong>URL:</strong> ${page.url}</p>
+#                                 <p><strong>Score:</strong> ${page.seo_score}/100</p>
+#                                 <p><strong>Title:</strong> ${page.title || 'No title'}</p>
+#                                 <p><strong>Meta Description:</strong> ${page.meta_description || 'No meta description'}</p>
+#                                 <p><strong>Word Count:</strong> ${page.word_count}</p>
+#                                 <p><strong>H1 Count:</strong> ${page.h1_count}</p>
+#                                 <p><strong>H2 Count:</strong> ${page.h2_count}</p>
+#                                 <p><strong>H3 Count:</strong> ${page.h3_count || 0}</p>
+#                                 <p><strong>Heading Order:</strong> ${page.heading_order || 'None'}</p>
+#                                 <p><strong>Missing Alt Tags:</strong> ${page.missing_alt_tags || 0} of ${page.total_images || 0} images</p>
+#                                 <p><strong>Total Images:</strong> ${page.total_images || 0}</p>
+#                                 <p><strong>Small Images (&lt;100px):</strong> ${page.small_images || 0}</p>
+#                                 <p><strong>Large Images (&gt;2000px):</strong> ${page.large_images || 0}</p>
+#                                 <p><strong>Ideal Images (100-2000px):</strong> ${page.ideal_images || 0}</p>
+#                                 <p><strong>Internal Links:</strong> ${page.internal_links || 0}</p>
+#                                 <p><strong>External Links:</strong> ${page.external_links || 0}</p>
+#                                 <p><strong>Canonical Tag:</strong> ${page.canonical_tag ? '✅ Yes' : '❌ No'}</p>
+#                                 <p><strong>Robots Meta:</strong> ${page.robots_meta || 'None'}</p>
+#                                 <p><strong>Viewport:</strong> ${page.viewport_present ? '✅ Yes' : '❌ No'}</p>
+#                                 <p><strong>Schema Types:</strong> ${page.schema_types || 'No schema found'}</p>
+#                                 <p><strong>OpenGraph Tags:</strong> ${page.opengraph_tags || 0}</p>
+#                                 <p><strong>Twitter Tags:</strong> ${page.twitter_tags || 0}</p>
+#                                 <p><strong>Readability Score:</strong> ${page.readability_score || 0}/100</p>
+#                                 <p><strong>Grammar Errors:</strong> ${page.grammar_errors || 0}</p>
+#                                 <p><strong>Text/HTML Ratio:</strong> ${page.text_to_html_ratio || 0}%</p>
+#                                 <p><strong>Top Keywords:</strong> ${page.top_keywords || 'None'}</p>
+#                                 <p><strong>Load Time:</strong> ${page.load_time || 0}ms</p>
+#                             </div>
+#                         `;
+#                     });
+#                     html += `</div>`;
+#                 }
+#             }
+            
+#             results.innerHTML = html;
+#             results.style.display = 'block';
+#         }
+        
+#         // Enter key support
+#         document.getElementById('url').addEventListener('keypress', function(e) {
+#             if (e.key === 'Enter') {
+#                 analyzeSEO();
+#             }
+#         });
+#     </script>
+# </body>
+# </html>
+# """
+
+# @app.get("/", response_class=HTMLResponse)
+# async def root():
+#     return HTML_TEMPLATE
+
+# @app.get("/health")
+# async def health_check():
+#     return {
+#         "status": "healthy", 
+#         "message": "SEO Analysis API is running",
+#         "timestamp": datetime.now().isoformat()
+#     }
+
+# @app.get("/analyze")
+# async def analyze_seo(
+#     url: str, 
+#     max_pages: int = 5, 
+#     use_ai: bool = True, 
+#     max_concurrent: int = 1
+# ):
+#     """
+#     Analyze a single website for SEO optimization
+    
+#     Args:
+#         url: Website URL to analyze
+#         max_pages: Maximum number of pages to analyze (default: 5)
+#         use_ai: Enable AI-powered suggestions (default: True)
+#         max_concurrent: Number of concurrent browsers (default: 1)
+#     """
+#     try:
+#         print(f"🔍 Starting SEO analysis for: {url}")
+        
+#         # Run SEO analysis
+#         results, csv_path = await run_seo_analysis_fastapi(
+#             base_url=url, 
+#             max_pages=max_pages,
+#             use_ai=use_ai,
+#             max_concurrent=max_concurrent,
+#             download=False
+#         )
+        
+#         # Calculate overall metrics
+#         avg_score = 0
+#         total_pages = len(results)
+        
+#         if results and 'error' not in results[0]:
+#             avg_score = round(sum(page.get("seo_score", 0) for page in results) / len(results), 1)
+        
+#         # Extract strengths and faults
+#         strengths = []
+#         faults = []
+#         suggestions = []
+        
+#         if results and 'error' not in results[0]:
+#             # Create complete page summaries for ALL pages with ALL metrics
+#             all_pages_summary = []
+#             for page in results:
+#                 page_summary = {
+#                     "url": page.get("url", ""),
+#                     "title": page.get("title", ""),
+#                     "meta_description": page.get("meta_description", ""),
+#                     "h1_count": page.get("h1_count", 0),
+#                     "h2_count": page.get("h2_count", 0),
+#                     "h3_count": page.get("h3_count", 0),
+#                     "heading_order": page.get("heading_order", ""),
+#                     "missing_alt_tags": page.get("missing_alt_tags", 0),
+#                     "total_images": page.get("total_images", 0),
+#                     "small_images": page.get("small_images", 0),
+#                     "large_images": page.get("large_images", 0),
+#                     "ideal_images": page.get("ideal_images", 0),
+#                     "internal_links": page.get("internal_links", 0),
+#                     "external_links": page.get("external_links", 0),
+#                     "canonical_tag": page.get("canonical_tag", False),
+#                     "robots_meta": page.get("robots_meta", ""),
+#                     "viewport_present": page.get("viewport_present", False),
+#                     "schema_types": page.get("schema_types", ""),
+#                     "opengraph_tags": page.get("opengraph_tags", 0),
+#                     "twitter_tags": page.get("twitter_tags", 0),
+#                     "word_count": page.get("word_count", 0),
+#                     "readability_score": page.get("readability_score", 0),
+#                     "grammar_errors": page.get("grammar_errors", 0),
+#                     "text_to_html_ratio": page.get("text_to_html_ratio", 0),
+#                     "top_keywords": page.get("top_keywords", ""),
+#                     "load_time": page.get("load_time", 0),
+#                     "seo_score": page.get("seo_score", 0)
+#                 }
+#                 all_pages_summary.append(page_summary)
+            
+#             # Sort pages by score (highest first)
+#             all_pages_summary.sort(key=lambda x: x["seo_score"], reverse=True)
+            
+#             # Analyze strengths
+#             good_titles = sum(1 for page in results if page.get('title') and 45 <= len(page['title']) <= 65)
+#             good_meta = sum(1 for page in results if page.get('meta_description') and 120 <= len(page['meta_description']) <= 155)
+#             mobile_friendly = sum(1 for page in results if page.get('viewport_present'))
+#             has_canonical = sum(1 for page in results if page.get('canonical_tag'))
+#             has_schema = sum(1 for page in results if page.get('schema_types') and page['schema_types'] != "No schema found")
+#             proper_h1 = sum(1 for page in results if page.get('h1_count', 0) == 1)
+#             good_images = sum(1 for page in results if page.get('missing_alt_tags', 0) == 0)
+#             good_wordcount = sum(1 for page in results if page.get('word_count', 0) >= 500)
+            
+#             if good_titles > 0:
+#                 strengths.append({
+#                     "title": "Optimized Title Tags",
+#                     "detail": f"{good_titles} pages have well-optimized title tags",
+#                     "impact": "high"
+#                 })
+            
+#             if good_meta > 0:
+#                 strengths.append({
+#                     "title": "Proper Meta Descriptions", 
+#                     "detail": f"{good_meta} pages have well-formatted meta descriptions",
+#                     "impact": "high"
+#                 })
+            
+#             if mobile_friendly > 0:
+#                 strengths.append({
+#                     "title": "Mobile Responsive",
+#                     "detail": f"{mobile_friendly} pages are mobile-friendly", 
+#                     "impact": "high"
+#                 })
+                
+#             if proper_h1 > 0:
+#                 strengths.append({
+#                     "title": "Proper H1 Structure",
+#                     "detail": f"{proper_h1} pages have correct H1 heading structure",
+#                     "impact": "high"
+#                 })
+
+#             # Analyze faults
+#             title_issues = sum(1 for page in results if not page.get('title') or len(page.get('title', '')) < 45 or len(page.get('title', '')) > 65)
+#             meta_issues = sum(1 for page in results if not page.get('meta_description') or len(page.get('meta_description', '')) < 120 or len(page.get('meta_description', '')) > 155)
+#             missing_alt = sum(page.get('missing_alt_tags', 0) for page in results)
+#             h1_issues = sum(1 for page in results if page.get('h1_count', 0) != 1)
+#             schema_issues = sum(1 for page in results if not page.get('schema_types') or page['schema_types'] == "No schema found")
+            
+#             if title_issues > 0:
+#                 faults.append({
+#                     "title": "Title Tag Issues",
+#                     "detail": f"{title_issues} pages need title optimization",
+#                     "severity": "high",
+#                     "pages": title_issues
+#                 })
+            
+#             if meta_issues > 0:
+#                 faults.append({
+#                     "title": "Meta Description Problems", 
+#                     "detail": f"{meta_issues} pages need meta description improvements",
+#                     "severity": "high",
+#                     "pages": meta_issues
+#                 })
+            
+#             if missing_alt > 0:
+#                 faults.append({
+#                     "title": "Missing Alt Text",
+#                     "detail": f"{missing_alt} images missing alt text",
+#                     "severity": "medium", 
+#                     "pages": sum(1 for page in results if page.get('missing_alt_tags', 0) > 0)
+#                 })
+                
+#             if h1_issues > 0:
+#                 faults.append({
+#                     "title": "H1 Structure Issues",
+#                     "detail": f"{h1_issues} pages have improper H1 structure", 
+#                     "severity": "high",
+#                     "pages": h1_issues
+#                 })
+
+#         # Extract AI suggestions
+#         if results and 'error' not in results[0] and use_ai:
+#             for page in results:
+#                 if page.get('ai_suggestions') and page['ai_suggestions'] not in [
+#                     "AI suggestions disabled - set valid OPENAI_API_KEY", 
+#                     "AI disabled - set valid OPENAI_API_KEY",
+#                     "AI suggestions disabled - set use_ai=True and OPENAI_API_KEY"
+#                 ]:
+#                     # Extract key suggestions from AI output
+#                     ai_text = page['ai_suggestions']
+                    
+#                     # Take first few lines as suggestions
+#                     lines = [line.strip() for line in ai_text.split('\n') if line.strip() and len(line.strip()) > 20]
+#                     for line in lines[:3]:
+#                         suggestions.append({
+#                             "title": "SEO Recommendation",
+#                             "detail": line[:150],
+#                             "priority": "high"
+#                         })
+
+#         # Limit suggestions
+#         suggestions = suggestions[:5]
+
+#         return {
+#             "status": "success",
+#             "url": url,
+#             "pages_analyzed": total_pages,
+#             "overall_score": avg_score,
+#             "strengths": strengths,
+#             "faults": faults, 
+#             "suggestions": suggestions,
+#             # Return ALL pages summary with ALL metrics
+#             "results_preview": all_pages_summary,
+#             "csv_download": f"/download/{os.path.basename(csv_path)}" if results and 'error' not in results[0] else None
+#         }
+        
+#     except Exception as e:
+#         print(f"❌ Analysis error: {e}")
+#         return {
+#             "status": "error", 
+#             "message": str(e)
+#         }
+
+# @app.get("/download/{filename}")
+# async def download_file(filename: str):
+#     """Download SEO analysis CSV report"""
+#     file_path = f"/tmp/{filename}"
+#     if os.path.exists(file_path):
+#         return FileResponse(
+#             file_path, 
+#             media_type='text/csv', 
+#             filename=f"seo_report.csv"
+#         )
+#     raise HTTPException(status_code=404, detail="File not found")
+
+# if __name__ == "__main__":
+#     import uvicorn
+#     print("🚀 Starting SEO Analysis API...")
+#     uvicorn.run(app, host="0.0.0.0", port=7860)

requirements.txt

@@ -0,0 +1,14 @@
+fastapi==0.104.1
+uvicorn==0.24.0
+beautifulsoup4==4.12.2
+aiohttp==3.9.1
+textstat==0.7.3
+playwright==1.40.0
+pandas==2.1.3
+lxml==4.9.3
+requests==2.31.0
+nest-asyncio==1.5.8
+language-tool-python==2.7
+python-multipart==0.0.6
+openai==0.28.1
+httpx==0.24.1

seo_analyzer.py

@@ -0,0 +1,1532 @@
+import os
+import csv
+import json
+import re
+import time
+import uuid
+import asyncio
+import aiohttp
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+from collections import Counter
+import textstat
+import concurrent.futures
+from dotenv import load_dotenv
+load_dotenv()
+
+# Apply nest_asyncio for compatibility
+import nest_asyncio
+nest_asyncio.apply()
+print("✅ nest_asyncio applied for compatibility")
+
+# Playwright imports
+from playwright.async_api import async_playwright
+PLAYWRIGHT_AVAILABLE = True
+
+# Optional grammar check
+try:
+    import language_tool_python
+    LT_AVAILABLE = True
+except Exception:
+    LT_AVAILABLE = False
+    print("⚠️ language_tool_python not available")
+
+HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
+
+# ==============================
+# SET YOUR OPENAI API KEY HERE
+# ==============================
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+if OPENAI_API_KEY:
+    print("✅ OpenAI API Key loaded from environment")
+else:
+    print("⚠️ OPENAI_API_KEY not set - AI features will be disabled")
+
+# ==============================
+# OPENAI CLIENT - COMPATIBLE VERSION (0.28.1)
+# ==============================
+_openai_client = None
+try:
+    import openai
+    OPENAI_AVAILABLE = True
+    _openai_client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+    print("✅ OpenAI client initialized successfully")
+except Exception as e:
+    OPENAI_AVAILABLE = False
+    print(f"⚠️ OpenAI not available: {e}")
+
+# ==============================
+# IMPROVED URL DISCOVERY WITH BETTER ERROR HANDLING
+# ==============================
+async def get_sitemap_links_parallel(base_url):
+    """Get all URLs from sitemap with parallel processing - IMPROVED"""
+    sitemap_urls = [
+        urljoin(base_url, "sitemap.xml"),
+        urljoin(base_url, "sitemap_index.xml"),
+        urljoin(base_url, "sitemap-0.xml"),
+        urljoin(base_url, "sitemap.txt"),
+        urljoin(base_url, "sitemap")
+    ]
+
+    async def fetch_sitemap(url):
+        try:
+            print(f"   Trying sitemap: {url}")
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, headers=HEADERS, timeout=15) as response:
+                    if response.status == 200:
+                        content_type = response.headers.get('content-type', '').lower()
+                        text = await response.text()
+
+                        # Check if it's XML sitemap
+                        if 'xml' in content_type or '<?xml' in text.lower():
+                            soup = BeautifulSoup(text, "xml")
+                            urls = [loc.text.strip() for loc in soup.find_all("loc") if loc.text.strip()]
+                            print(f"   ✅ Found {len(urls)} URLs in {url}")
+                            return urls
+                        # Check if it's text sitemap
+                        elif 'text/plain' in content_type or '\n' in text:
+                            urls = [line.strip() for line in text.split('\n') if line.strip() and line.startswith('http')]
+                            print(f"   ✅ Found {len(urls)} URLs in text sitemap")
+                            return urls
+        except Exception as e:
+            print(f"   ❌ Sitemap {url} failed: {str(e)[:100]}")
+            return []
+
+    # Try all sitemap URLs in parallel
+    tasks = [fetch_sitemap(url) for url in sitemap_urls]
+    results = await asyncio.gather(*tasks)
+
+    # Flatten and deduplicate
+    all_urls = set()
+    for url_list in results:
+        if url_list:  # Only extend if not None or empty
+            all_urls.update(url_list)
+
+    return list(all_urls)
+
+async def discover_urls_parallel(base_url, max_pages=20):
+    """Discover URLs through sitemap and light crawling - IMPROVED ERROR HANDLING"""
+    print("🔍 Discovering URLs...")
+
+    try:
+        # Get sitemap URLs first (fastest)
+        sitemap_urls = await get_sitemap_links_parallel(base_url)
+
+        if sitemap_urls:
+            print(f"📄 Found {len(sitemap_urls)} URLs in sitemap")
+
+            # FILTER OUT XML FILES - ONLY KEEP HTML PAGES
+            html_urls = []
+            for url in sitemap_urls:
+                # Skip XML files, sitemaps, RSS feeds, etc.
+                if any(xml_pattern in url.lower() for xml_pattern in ['.xml', 'sitemap', 'rss', 'feed']):
+                    continue
+                # Skip other non-HTML files
+                if any(non_html in url.lower() for non_html in ['.pdf', '.jpg', '.png', '.gif', '.css', '.js']):
+                    continue
+                html_urls.append(url)
+
+            print(f"🔧 Filtered to {len(html_urls)} HTML pages (removed {len(sitemap_urls) - len(html_urls)} non-HTML files)")
+
+            if html_urls:
+                return html_urls[:max_pages]
+            else:
+                print("⚠️ No HTML pages found in sitemap, falling back to homepage crawl...")
+
+        # Fallback: light crawl from homepage - ONLY GET HTML PAGES
+        print("🕷️ No valid HTML pages in sitemap, crawling from homepage...")
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context()
+            page = await context.new_page()
+
+            print(f"   Navigating to {base_url}...")
+            await page.goto(base_url, wait_until='domcontentloaded', timeout=30000)
+
+            # Extract all internal links - FILTER FOR HTML PAGES
+            links = await page.evaluate("""(baseDomain) => {
+                const allLinks = Array.from(document.links)
+                    .map(link => link.href)
+                    .filter(href => href && href.includes(baseDomain))
+                    .filter(href => !href.includes('#') && !href.includes('javascript:'))
+                    // FILTER OUT NON-HTML FILES
+                    .filter(href => !href.includes('.xml') && !href.includes('sitemap') && !href.includes('rss') && !href.includes('feed'))
+                    .filter(href => !href.match(/\\.(pdf|jpg|png|gif|css|js|json)$/i))
+                    .slice(0, 50);
+                return [...new Set(allLinks)]; // Remove duplicates
+            }""", urlparse(base_url).netloc)
+
+            await browser.close()
+
+            if links:
+                print(f"   Found {len(links)} internal HTML links")
+                result_urls = [base_url] + links[:max_pages-1]
+                return result_urls
+            else:
+                print("   No internal links found, using only homepage")
+                return [base_url]
+
+    except Exception as e:
+        print(f"❌ URL discovery failed: {e}")
+        print("   Using fallback: homepage only")
+        return [base_url]  # Always return at least homepage
+
+# ==============================
+# PARALLEL PLAYWRIGHT FETCHER
+# ==============================
+async def fetch_page_playwright(url, browser, timeout=25000):
+    """Fetch a single page with Playwright"""
+    context = await browser.new_context(
+        viewport={'width': 1920, 'height': 1080},
+        user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+        java_script_enabled=True
+    )
+
+    # Block unnecessary resources for speed
+    await context.route("**/*.{png,jpg,jpeg,gif,svg,webp}", lambda route: route.abort())
+    await context.route("**/*.css", lambda route: route.abort())
+
+    page = await context.new_page()
+
+    try:
+        # Longer timeout
+        print(f"   📡 Fetching: {url}")
+        await page.goto(url, wait_until='domcontentloaded', timeout=timeout)
+
+        # Wait for critical content
+        await page.wait_for_selector('body', timeout=15000)
+
+        # Extract comprehensive SEO data
+        seo_data = await page.evaluate("""() => {
+            // Get all meta tags
+            const metas = {};
+            document.querySelectorAll('meta').forEach(meta => {
+                const name = meta.getAttribute('name') || meta.getAttribute('property');
+                if (name) metas[name] = meta.getAttribute('content');
+            });
+
+            // Get all images with detailed info
+            const images = Array.from(document.images).map(img => ({
+                src: img.src,
+                alt: img.alt || '',
+                naturalWidth: img.naturalWidth,
+                naturalHeight: img.naturalHeight,
+                complete: img.complete
+            }));
+
+            // Get all links
+            const links = Array.from(document.links).map(link => ({
+                href: link.href,
+                text: link.textContent?.slice(0, 100) || '',
+                rel: link.rel
+            }));
+
+            // Get schema data
+            const schemas = [];
+            document.querySelectorAll('script[type="application/ld+json"]').forEach(script => {
+                try {
+                    if (script.textContent) {
+                        const data = JSON.parse(script.textContent);
+                        schemas.push(data);
+                    }
+                } catch (e) {}
+            });
+
+            return {
+                url: window.location.href,
+                title: document.title,
+                metas: metas,
+                description: metas.description || metas['og:description'] || '',
+                canonical: document.querySelector('link[rel="canonical"]')?.href || '',
+                robots: metas.robots || '',
+                viewport: metas.viewport || '',
+                h1_count: document.querySelectorAll('h1').length,
+                h2_count: document.querySelectorAll('h2').length,
+                h3_count: document.querySelectorAll('h3').length,
+                images: images,
+                links: links,
+                schemas: schemas,
+                html: document.documentElement.outerHTML,
+                opengraph: {
+                    title: metas['og:title'] || '',
+                    description: metas['og:description'] || '',
+                    image: metas['og:image'] || '',
+                    url: metas['og:url'] || ''
+                },
+                twitter: {
+                    title: metas['twitter:title'] || '',
+                    description: metas['twitter:description'] || '',
+                    image: metas['twitter:image'] || '',
+                    card: metas['twitter:card'] || ''
+                }
+            };
+        }""")
+
+        print(f"   ✅ Success: {url}")
+        return seo_data
+
+    except Exception as e:
+        print(f"   ❌ Failed: {url} - {str(e)[:100]}...")
+        return None
+    finally:
+        await context.close()
+
+async def fetch_all_pages_parallel(urls, max_concurrent=1):
+    """Fetch multiple pages in parallel"""
+    if not urls:
+        print("❌ No URLs to fetch!")
+        return []
+
+    print(f"🚀 Launching {max_concurrent} browsers for parallel fetching...")
+    print(f"📡 Fetching {len(urls)} pages...")
+
+    async with async_playwright() as p:
+        # Launch browser with optimized settings
+        browser = await p.chromium.launch(
+            headless=True,
+            args=[
+                '--disable-gpu',
+                '--disable-dev-shm-usage',
+                '--disable-setuid-sandbox',
+                '--no-first-run',
+                '--no-sandbox',
+                '--no-zygote',
+                '--deterministic-fetch',
+                '--max_old_space_size=4096'
+            ]
+        )
+
+        # Create semaphore for concurrency control
+        semaphore = asyncio.Semaphore(max_concurrent)
+
+        async def fetch_with_semaphore(url):
+            async with semaphore:
+                return await fetch_page_playwright(url, browser)
+
+        # Fetch all pages in parallel with progress
+        tasks = [fetch_with_semaphore(url) for url in urls]
+        results = []
+
+        for i, task in enumerate(asyncio.as_completed(tasks)):
+            result = await task
+            results.append(result)
+            if (i + 1) % 2 == 0 or (i + 1) == len(urls):
+                print(f"   📊 Progress: {i + 1}/{len(urls)} pages completed")
+
+        await browser.close()
+
+        # Filter out failed fetches
+        successful_results = [r for r in results if r is not None]
+        print(f"✅ Successfully fetched {len(successful_results)} out of {len(urls)} pages")
+
+        return successful_results
+
+# ==============================
+# PARALLEL SEO ANALYSIS - FIXED SCHEMA EXTRACTION
+# ==============================
+async def analyze_pages_parallel(playwright_data_list, domain):
+    """Analyze all pages in parallel"""
+    if not playwright_data_list:
+        print("❌ No data to analyze!")
+        return []
+
+    print("🔬 Analyzing pages in parallel...")
+
+    def analyze_single_page(seo_data):
+        """Analyze a single page's SEO data"""
+        try:
+            html = seo_data.get('html', '')
+            soup = BeautifulSoup(html, 'html.parser')
+
+            # Extract text content
+            text = soup.get_text(separator=" ", strip=True)
+
+            # Images analysis
+            images_data = seo_data.get('images', [])
+            total_images = len(images_data)
+            missing_alt = len([img for img in images_data if not img.get('alt')])
+
+            # Links analysis
+            links_data = seo_data.get('links', [])
+            internal_links = len([link for link in links_data if domain in link.get('href', '')])
+            external_links = len([link for link in links_data if domain not in link.get('href', '')])
+
+            # ==============================
+            # FIXED SCHEMA EXTRACTION - PROPERLY INDENTED
+            # ==============================
+            schemas = seo_data.get('schemas', [])
+            schema_types = []
+
+            for schema in schemas:
+                try:
+                    # Handle different schema formats
+                    if isinstance(schema, dict):
+                        # Direct schema object
+                        if '@type' in schema:
+                            schema_types.append(schema['@type'])
+                        # Schema with @graph
+                        if '@graph' in schema and isinstance(schema['@graph'], list):
+                            for item in schema['@graph']:
+                                if isinstance(item, dict) and '@type' in item:
+                                    schema_types.append(item['@type'])
+                    elif isinstance(schema, list):
+                        # Array of schemas
+                        for item in schema:
+                            if isinstance(item, dict) and '@type' in item:
+                                schema_types.append(item['@type'])
+                except Exception as e:
+                    print(f"   Schema parsing error: {e}")
+
+            # Also check for microdata and other schema formats in HTML
+            try:
+                # Check for microdata
+                microdata = soup.find_all(attrs={"itemtype": True})
+                for item in microdata:
+                    itemtype = item.get('itemtype', '')
+                    if itemtype:
+                        schema_types.append(itemtype.split('/')[-1])  # Get just the type name
+
+                # Check for other schema script tags
+                schema_scripts = soup.find_all('script', type=lambda x: x and 'ld+json' in x)
+                for script in schema_scripts:
+                    try:
+                        if script.string:
+                            data = json.loads(script.string)
+                            if isinstance(data, dict) and '@type' in data:
+                                schema_types.append(data['@type'])
+                            elif isinstance(data, list):
+                                for item in data:
+                                    if isinstance(item, dict) and '@type' in item:
+                                        schema_types.append(item['@type'])
+                    except:
+                        pass
+
+            except Exception as e:
+                print(f"   HTML schema extraction error: {e}")
+
+            # Deduplicate schema types
+            schema_types = list(set(schema_types))
+            # ==============================
+            # END OF FIXED SCHEMA EXTRACTION
+            # ==============================
+
+            # Metrics
+            try:
+                readability_score = textstat.flesch_reading_ease(text)
+            except Exception:
+                readability_score = 0
+
+            word_count = len(text.split())
+
+            # Grammar errors (optional)
+            grammar_errors = 0
+            if LT_AVAILABLE:
+                try:
+                    tool = language_tool_python.LanguageTool('en-US')
+                    grammar_errors = len(tool.check(text[:1000]))
+                    tool.close()
+                except Exception:
+                    pass
+
+            # Keyword density
+            top_keywords = keyword_density(text)
+            text_to_html_ratio = round((len(text) / len(html)) * 100, 2) if html else 0
+
+            # Compile page data
+            page = {
+                "url": seo_data.get('url', ''),
+                "title": seo_data.get('title', ''),
+                "meta_description": seo_data.get('description', ''),
+                "h1_count": seo_data.get('h1_count', 0),
+                "h2_count": seo_data.get('h2_count', 0),
+                "h3_count": seo_data.get('h3_count', 0),
+                "heading_order": get_heading_order(soup),
+                "missing_alt_tags": missing_alt,
+                "total_images": total_images,
+                "small_images": len([img for img in images_data if img.get('naturalWidth', 0) < 100]),
+                "large_images": len([img for img in images_data if img.get('naturalWidth', 0) > 2000]),
+                "ideal_images": len([img for img in images_data if 100 <= img.get('naturalWidth', 0) <= 2000]),
+                "internal_links": internal_links,
+                "external_links": external_links,
+                "canonical_tag": bool(seo_data.get('canonical')),
+                "robots_meta": seo_data.get('robots', ''),
+                "viewport_present": 'width' in seo_data.get('viewport', ''),
+                "schema_types": ", ".join(schema_types) if schema_types else "No schema found",
+                "opengraph_tags": count_opengraph_tags(seo_data.get('metas', {})),
+                "twitter_tags": count_twitter_tags(seo_data.get('metas', {})),
+                "word_count": word_count,
+                "readability_score": readability_score,
+                "grammar_errors": grammar_errors,
+                "text_to_html_ratio": text_to_html_ratio,
+                "top_keywords": top_keywords,
+                "load_time": 0,
+            }
+
+            return page
+
+        except Exception as e:
+            print(f"❌ Analysis error for {seo_data.get('url', 'unknown')}: {e}")
+            return None
+
+    # Run analysis in parallel using ThreadPoolExecutor
+    loop = asyncio.get_event_loop()
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        tasks = [
+            loop.run_in_executor(executor, analyze_single_page, data)
+            for data in playwright_data_list
+        ]
+        results = await asyncio.gather(*tasks)
+
+    # Filter out failed analyses
+    successful_results = [r for r in results if r is not None]
+    print(f"✅ Successfully analyzed {len(successful_results)} pages")
+    return successful_results
+
+def get_heading_order(soup):
+    """Extract heading order from BeautifulSoup"""
+    headings = soup.find_all(re.compile('^h[1-6]$'))
+    return ", ".join([h.name for h in headings])
+
+def count_opengraph_tags(metas):
+    """Count OpenGraph tags"""
+    return len([k for k in metas.keys() if k.startswith('og:')])
+
+def count_twitter_tags(metas):
+    """Count Twitter card tags"""
+    return len([k for k in metas.keys() if k.startswith('twitter:')])
+
+def keyword_density(text):
+    """Calculate keyword density"""
+    words = re.findall(r'\b\w+\b', (text or "").lower())
+    freq = Counter(w for w in words if len(w) > 3)
+    total = sum(freq.values()) or 1
+    items = sorted([(k, round(v / total * 100, 2)) for k, v in freq.items() if v > 1],
+                   key=lambda x: -x[1])[:10]
+    return ", ".join([f"{k}:{p}%" for k, p in items])
+
+# ==============================
+# ULTRA-SPECIFIC AI SUGGESTIONS - WITH EXACT REPLACEMENTS
+# ==============================
+async def generate_page_suggestions_async(page_data):
+    """Generate ULTRA-SPECIFIC AI suggestions with exact replacements"""
+    api_key = os.environ.get("OPENAI_API_KEY")
+
+    if not api_key or api_key == "YOUR_OPENAI_API_KEY_HERE" or not OPENAI_AVAILABLE:
+        return "AI disabled - set valid OPENAI_API_KEY"
+
+    # Extract detailed page data
+    url = page_data.get('url', 'Unknown URL')
+    title = page_data.get('title', '')
+    meta_description = page_data.get('meta_description', '')
+    seo_score = page_data.get('seo_score', 0)
+    h1_count = page_data.get('h1_count', 0)
+    h2_count = page_data.get('h2_count', 0)
+    h3_count = page_data.get('h3_count', 0)
+    word_count = page_data.get('word_count', 0)
+    readability_score = page_data.get('readability_score', 0)
+    missing_alt_tags = page_data.get('missing_alt_tags', 0)
+    total_images = page_data.get('total_images', 0)
+    schema_types = page_data.get('schema_types', '')
+    internal_links = page_data.get('internal_links', 0)
+    external_links = page_data.get('external_links', 0)
+    opengraph_tags = page_data.get('opengraph_tags', 0)
+    twitter_tags = page_data.get('twitter_tags', 0)
+    top_keywords = page_data.get('top_keywords', '')
+    heading_order = page_data.get('heading_order', '')
+
+    # ULTRA-SPECIFIC PROMPT - Demands exact replacements
+    prompt = f"""
+You are an expert technical SEO consultant. Analyze this page and provide EXACT, ACTIONABLE recommendations with SPECIFIC REPLACEMENTS.
+
+CRITICAL REQUIREMENTS:
+- Provide EXACT replacement text for bad titles, meta descriptions, etc.
+- Give SPECIFIC OpenGraph and Twitter Card markup when missing
+- Provide EXACT schema markup code when missing
+- Give SPECIFIC H1 text when missing
+- Provide EXACT alt text examples for images
+
+PAGE DATA:
+URL: {url}
+Current SEO Score: {seo_score}/100
+
+CURRENT CONTENT:
+- Title: "{title}" ({len(title)} chars)
+- Meta Description: "{meta_description}" ({len(meta_description)} chars)
+- H1 Count: {h1_count} | H2 Count: {h2_count} | H3 Count: {h3_count}
+- Word Count: {word_count} words
+- Readability: {readability_score}/100
+- Missing Alt Tags: {missing_alt_tags} of {total_images} images
+- Schema: {schema_types}
+- Internal Links: {internal_links} | External Links: {external_links}
+- OpenGraph Tags: {opengraph_tags} | Twitter Cards: {twitter_tags}
+- Top Keywords: {top_keywords}
+- Heading Structure: {heading_order}
+
+Provide recommendations in this EXACT format:
+
+HIGH IMPACT:
+1. TITLE OPTIMIZATION:
+   Current: "{title}" ({len(title)} chars)
+   REPLACE WITH: "[Exact new title text - 55-60 characters]"
+
+2. META DESCRIPTION:
+   Current: "{meta_description}" ({len(meta_description)} chars)
+   REPLACE WITH: "[Exact new meta description - 150-155 characters]"
+
+3. H1 TAG:
+   Current: {h1_count} H1 tags
+   ADD THIS EXACT H1: "[Exact H1 text with primary keyword]"
+
+MEDIUM IMPACT:
+4. OPENGRAPH TAGS (Missing {8 - opengraph_tags} tags):
+   ADD THIS EXACT MARKUP:
+   <meta property="og:title" content="[Exact og:title]">
+   <meta property="og:description" content="[Exact og:description]">
+   <meta property="og:image" content="[Suggested image URL]">
+   <meta property="og:url" content="{url}">
+
+5. TWITTER CARDS (Missing {5 - twitter_tags} tags):
+   ADD THIS EXACT MARKUP:
+   <meta name="twitter:title" content="[Exact twitter:title]">
+   <meta name="twitter:description" content="[Exact twitter:description]">
+   <meta name="twitter:image" content="[Suggested image URL]">
+   <meta name="twitter:card" content="summary_large_image">
+
+6. SCHEMA MARKUP:
+   Current: {schema_types}
+   ADD THIS EXACT SCHEMA:
+   [Provide complete JSON-LD schema code]
+
+LOW IMPACT:
+7. IMAGE ALT TEXT:
+   Missing alt text for {missing_alt_tags} images
+   EXAMPLE ALT TEXTS:
+   - "[Exact alt text for first image]"
+   - "[Exact alt text for second image]"
+
+8. CONTENT IMPROVEMENT:
+   Current: {word_count} words, {readability_score}/100 readability
+   ADD THIS EXACT CONTENT SECTION:
+   "[Specific content to add with exact paragraph]"
+
+Provide EXACT text replacements - no generic advice!
+"""
+
+    try:
+        print(f"   🤖 Generating ULTRA-SPECIFIC suggestions for: {url[:50]}...")
+
+        # gpt-5-nano is a reasoning-family model: it only accepts
+        # max_completion_tokens (not max_tokens) and only the default
+        # temperature (1), so no temperature override is passed.
+        # Uses OpenAI 0.28.1 syntax (matches installed SDK version).
+        # response = await asyncio.get_event_loop().run_in_executor(
+        #     None,
+        #     lambda: openai.ChatCompletion.create(
+        #         model="gpt-5-nano",
+        #         messages=[
+        #             {"role": "system", "content": "You are a technical SEO expert who provides EXACT replacement text and markup. Always give specific examples and complete code snippets. No generic advice allowed."},
+        #             {"role": "user", "content": prompt}
+        #         ],
+        #         max_completion_tokens=800,
+        #     )
+        # )
+        response = await asyncio.get_event_loop().run_in_executor(
+    None,
+    lambda: _openai_client.chat.completions.create(
+        model="gpt-5-nano",
+        messages=[
+            {"role": "system", "content": "You are a technical SEO expert who provides EXACT replacement text and markup. Always give specific examples and complete code snippets. No generic advice allowed."},
+            {"role": "user", "content": prompt}
+        ],
+        max_completion_tokens=6000,
+        reasoning_effort="low",
+    )
+)
+
+        ai_suggestion = response.choices[0].message.content.strip()
+        print(f"   ✅ ULTRA-SPECIFIC suggestions generated for: {url[:50]}...")
+        return ai_suggestion
+
+    except Exception as e:
+        print(f"   ❌ AI failed for {url[:50]}: {str(e)[:100]}")
+        return generate_ultra_specific_fallback(page_data)
+
+def generate_ultra_specific_fallback(page_data):
+    """Generate ultra-specific fallback suggestions with exact examples"""
+    suggestions = []
+
+    url = page_data.get('url', '')
+    title = page_data.get('title', '')
+    meta_desc = page_data.get('meta_description', '')
+    h1_count = page_data.get('h1_count', 0)
+    word_count = page_data.get('word_count', 0)
+    missing_alt = page_data.get('missing_alt_tags', 0)
+    schema_types = page_data.get('schema_types', '')
+    opengraph_tags = page_data.get('opengraph_tags', 0)
+    twitter_tags = page_data.get('twitter_tags', 0)
+
+    # Extract domain for context
+    domain = urlparse(url).netloc.replace('www.', '')
+    site_name = domain.split('.')[0].title()
+
+    # HIGH IMPACT - EXACT REPLACEMENTS
+    if h1_count == 0:
+        page_topic = url.split('/')[-1].replace('-', ' ').title()
+        suggestions.append(f"HIGH: ADD EXACT H1: '{page_topic} - Complete Guide | {site_name}'")
+
+    title_len = len(title)
+    if title_len < 45 or title_len > 65:
+        page_topic = url.split('/')[-1].replace('-', ' ').title()
+        suggestions.append(f"HIGH: REPLACE TITLE: '{page_topic} - Complete {site_name} Guide 2024'")
+
+    if not meta_desc or len(meta_desc) < 50:
+        page_topic = url.split('/')[-1].replace('-', ' ').title()
+        suggestions.append(f"HIGH: REPLACE META: 'Learn everything about {page_topic.lower()} with our complete guide. Get expert tips, best practices, and step-by-step instructions from {site_name}.'")
+
+    # MEDIUM IMPACT - EXACT MARKUP
+    if opengraph_tags < 4:
+        page_topic = url.split('/')[-1].replace('-', ' ').title()
+        suggestions.append(f"MEDIUM: ADD OPENGraph:\n<meta property=\"og:title\" content=\"{page_topic} - {site_name}\">\n<meta property=\"og:description\" content=\"Complete guide to {page_topic.lower()} with expert insights\">\n<meta property=\"og:image\" content=\"https://{domain}/images/{page_topic.lower().replace(' ', '-')}.jpg\">")
+
+    if twitter_tags < 3:
+        page_topic = url.split('/')[-1].replace('-', ' ').title()
+        suggestions.append(f"MEDIUM: ADD TWITTER CARDS:\n<meta name=\"twitter:title\" content=\"{page_topic} Guide\">\n<meta name=\"twitter:description\" content=\"Master {page_topic.lower()} with {site_name}'s expert guide\">\n<meta name=\"twitter:card\" content=\"summary_large_image\">")
+
+    if schema_types == "No schema found":
+        page_topic = url.split('/')[-1].replace('-', ' ').title()
+        suggestions.append(f"MEDIUM: ADD SCHEMA:\n<script type=\"application/ld+json\">\n{{\n  \"@context\": \"https://schema.org\",\n  \"@type\": \"Article\",\n  \"headline\": \"{page_topic} Complete Guide\",\n  \"description\": \"Expert guide to {page_topic.lower()} with best practices\",\n  \"author\": {{\n    \"@type\": \"Organization\",\n    \"name\": \"{site_name}\"\n  }}\n}}\n</script>")
+
+    # LOW IMPACT - EXACT EXAMPLES
+    if missing_alt > 0:
+        page_topic = url.split('/')[-1].replace('-', ' ').title()
+        suggestions.append(f"LOW: ADD ALT TEXTS:\n- \"{page_topic} diagram and explanation\"\n- \"Step-by-step {page_topic.lower()} process visualization\"\n- \"{site_name} {page_topic} tutorial screenshot\"")
+
+    if word_count < 800:
+        suggestions.append(f"LOW: ADD CONTENT SECTION:\n\"In this comprehensive guide, we'll cover the essential aspects of {page_topic.lower()} including best practices, common pitfalls to avoid, and actionable strategies you can implement immediately. Whether you're a beginner or looking to advanced your skills, this guide provides the foundation you need for success.\"")
+
+    return "\n\n".join(suggestions) if suggestions else "All major elements optimized - focus on internal linking and user experience"
+
+async def generate_all_page_suggestions_parallel(pages):
+    """Generate ULTRA-SPECIFIC AI suggestions for ALL pages in parallel"""
+    suggestions = {}
+
+    print(f"🤖 Generating ULTRA-SPECIFIC AI suggestions for {len(pages)} pages in parallel...")
+
+    # Create tasks for ALL pages
+    tasks = []
+    for i, page in enumerate(pages):
+        task = generate_page_suggestions_async(page)
+        tasks.append((i, task))
+
+    # Run ALL AI calls concurrently
+    if tasks:
+        coroutines = [task for _, task in tasks]
+        results = await asyncio.gather(*coroutines, return_exceptions=True)
+
+        # Map results back to pages
+        for result_idx, (page_idx, _) in enumerate(tasks):
+            result = results[result_idx]
+            if isinstance(result, Exception):
+                print(f"   ❌ AI failed for page {page_idx}, using ultra-specific fallback")
+                suggestions[page_idx] = generate_ultra_specific_fallback(pages[page_idx])
+            else:
+                suggestions[page_idx] = result
+
+    return suggestions
+
+async def add_comprehensive_suggestions_async(results):
+    """Add comprehensive ULTRA-SPECIFIC AI suggestions to all pages"""
+    api_key = os.environ.get("OPENAI_API_KEY")
+
+    if not api_key or api_key == "YOUR_OPENAI_API_KEY_HERE" or not OPENAI_AVAILABLE:
+        print("⚠️ OPENAI_API_KEY not set — AI suggestions disabled.")
+        for p in results:
+            p["ai_suggestions"] = "AI suggestions disabled - set valid OPENAI_API_KEY"
+        return
+
+    print("🚀 Generating comprehensive ULTRA-SPECIFIC AI suggestions for all pages...")
+
+    # Get ULTRA-SPECIFIC AI suggestions
+    suggestions_dict = await generate_all_page_suggestions_parallel(results)
+
+    # Apply suggestions to pages
+    for i, p in enumerate(results):
+        ai_suggestion = suggestions_dict.get(i, "No AI suggestions generated")
+        p["ai_suggestions"] = ai_suggestion
+
+
+# ==============================
+# DETECT PAGE TYPE FUNCTION
+# ==============================
+def detect_page_type(url, page_data):
+    """Detect if page is homepage, article, category, etc."""
+    parsed = urlparse(url)
+    path = parsed.path.strip('/')
+
+    # Check if it's homepage
+    if not path or path == '' or path == 'index.html' or path == 'index.php':
+        return 'homepage'
+
+    # Check for common article patterns
+    article_patterns = ['/blog/', '/article/', '/news/', '/post/', '/2024/', '/2025/']
+    if any(pattern in url for pattern in article_patterns):
+        return 'article'
+
+    # Check for category/listing pages
+    category_patterns = ['/category/', '/tag/', '/topic/']
+    if any(pattern in url for pattern in category_patterns):
+        return 'category'
+
+    # Default
+    return 'standard'
+# ==============================
+# COMPREHENSIVE SCORING FUNCTION - FIXED VERSION
+# ==============================
+def calculate_seo_score(page):
+    """Calculate comprehensive SEO score with realistic thresholds"""
+    score = 0
+    max_score = 100
+
+    # Detect page type once for use throughout
+    url = page.get('url', '')
+    page_type = detect_page_type(url, page)
+
+    # ===== TITLE OPTIMIZATION (10 points) =====
+    title = page.get('title', '')
+    if title:
+        title_len = len(title)
+        if 50 <= title_len <= 60:  # Perfect
+            score += 10
+        elif 45 <= title_len <= 65:  # Good
+            score += 8
+        elif 30 <= title_len <= 70:  # Acceptable
+            score += 6
+        elif title_len > 0:  # Exists but poor
+            score += 3
+
+    # ===== META DESCRIPTION (8 points) =====
+    meta_desc = page.get('meta_description', '')
+    if meta_desc:
+        meta_len = len(meta_desc)
+        if 120 <= meta_len <= 155:  # Perfect
+            score += 8
+        elif 100 <= meta_len <= 160:  # Good
+            score += 6
+        elif 70 <= meta_len <= 170:  # Acceptable
+            score += 4
+        elif meta_len > 0:  # Exists but poor
+            score += 2
+
+    # ===== HEADING STRUCTURE (12 points) =====
+    h1_count = page.get('h1_count', 0)
+    heading_order = page.get('heading_order', '')
+
+    # H1 Score (6 points)
+    if h1_count == 1:  # Perfect
+        score += 6
+    elif h1_count == 0:  # Critical
+        score += 0
+    elif h1_count == 2:  # Minor issue
+        score += 4
+    else:  # Multiple H1s
+        score += 1
+
+    # FIXED: Heading Hierarchy (6 points) - More flexible
+    if heading_order:
+        headings = [h.strip() for h in heading_order.split(',')]
+        heading_levels = []
+
+        for heading in headings:
+            if heading.startswith('h'):
+                try:
+                    level = int(heading[1])
+                    heading_levels.append(level)
+                except:
+                    continue
+
+        # More flexible heading structure scoring
+        has_h1 = 1 in heading_levels
+        has_h2 = 2 in heading_levels
+        has_h3 = 3 in heading_levels
+
+        # Check if headings follow a logical order
+        if has_h1 and (has_h2 or has_h3):
+            score += 6  # Full points for logical structure
+        elif has_h1:
+            score += 4  # Has H1 but no subheadings
+        elif has_h2 or has_h3:
+            score += 2  # No H1 but has other headings
+
+    # ===== IMAGE OPTIMIZATION (15 points) =====
+    total_images = page.get('total_images', 0)
+    missing_alt_tags = page.get('missing_alt_tags', 0)
+    small_images = page.get('small_images', 0)
+    large_images = page.get('large_images', 0)
+    ideal_images = page.get('ideal_images', 0)
+
+    # Alt Text Score (5 points)
+    if total_images == 0:
+        score += 5
+    else:
+        alt_ratio = (total_images - missing_alt_tags) / total_images
+        if alt_ratio >= 0.95:
+            score += 5
+        elif alt_ratio >= 0.80:
+            score += 4
+        elif alt_ratio >= 0.60:
+            score += 3
+        elif alt_ratio >= 0.40:
+            score += 2
+        elif alt_ratio > 0:
+            score += 1
+
+    # FIXED: Image Size Optimization (5 points) - Forgive small images
+    if total_images > 0:
+        # Don't penalize small images that might be icons/logos
+        # Assume first 2 small images could be logo/favicon/icons
+        forgiven_small_images = max(0, small_images - 2)
+        adjusted_total = total_images - forgiven_small_images
+
+        if adjusted_total > 0:
+            # Recalculate ideal ratio without penalized small images
+            ideal_ratio = ideal_images / adjusted_total if adjusted_total > 0 else 1
+
+            if ideal_ratio >= 0.7:  # Slightly lowered threshold
+                score += 5
+            elif ideal_ratio >= 0.5:
+                score += 4
+            elif ideal_ratio >= 0.3:
+                score += 3
+            elif ideal_ratio >= 0.15:
+                score += 2
+            else:
+                score += 1
+        else:
+            score += 5  # All images are forgiven (probably icons/logos)
+    else:
+        score += 5
+
+    # Image Quantity (5 points)
+    if total_images == 0:
+        score += 3
+    elif 3 <= total_images <= 20:
+        score += 5
+    elif total_images <= 50:
+        score += 4
+    elif total_images <= 100:
+        score += 3
+    else:
+        score += 2
+
+    # ===== FIXED LINK STRUCTURE (10 points) =====
+    internal_links = page.get('internal_links', 0)
+    external_links = page.get('external_links', 0)
+
+    # Internal Links (6 points) - More realistic thresholds
+    if internal_links >= 30:  # Was 80
+        score += 6
+    elif internal_links >= 20:  # Was 60
+        score += 5
+    elif internal_links >= 10:  # Was 40
+        score += 4
+    elif internal_links >= 5:   # Was 20
+        score += 3
+    elif internal_links >= 3:   # Was 10
+        score += 2
+    elif internal_links >= 1:
+        score += 1
+
+    # External Links (4 points) - Keep as is
+    if external_links >= 10:
+        score += 4
+    elif external_links >= 7:
+        score += 3
+    elif external_links >= 4:
+        score += 2
+    elif external_links >= 1:
+        score += 1
+
+    # ===== TECHNICAL SEO (20 points) =====
+    # Canonical Tag (3 points)
+    if page.get('canonical_tag', False):
+        score += 3
+
+    # Robots Meta (3 points)
+    robots_meta = page.get('robots_meta', '')
+    if robots_meta:
+        robots_lower = robots_meta.lower()
+        if 'noindex' not in robots_lower and 'nofollow' not in robots_lower:
+            score += 3
+        elif 'noindex' in robots_lower:
+            score += 0
+        else:
+            score += 2
+    else:
+        score += 2
+
+    # Viewport (3 points)
+    if page.get('viewport_present', False):
+        score += 3
+
+    # FIXED: Schema Markup (4 points) - Quality over quantity
+    schema_types = page.get('schema_types', '')
+    if schema_types and schema_types.strip() and schema_types != "No schema found":
+        schema_count = len([s for s in schema_types.split(', ') if s.strip()])
+
+        # Check for important schema types
+        important_schemas = ['Organization', 'WebSite', 'Article', 'Product', 'LocalBusiness']
+        has_important = any(schema in schema_types for schema in important_schemas)
+
+        if has_important and schema_count >= 2:
+            score += 4  # Has important schema plus others
+        elif has_important:
+            score += 3  # Has at least one important schema
+        elif schema_count >= 2:
+            score += 3  # Multiple schemas even if not "important"
+        elif schema_count == 1:
+            score += 2  # Has some schema
+    else:
+        # No schema, but don't penalize too heavily for certain page types
+        if page_type not in ['article', 'product']:  # Pages that really should have schema
+            score += 1  # Small penalty instead of zero
+
+    # OpenGraph Tags (4 points)
+    opengraph_tags = page.get('opengraph_tags', 0)
+    if opengraph_tags >= 10:
+        score += 4
+    elif opengraph_tags >= 7:
+        score += 3
+    elif opengraph_tags >= 5:
+        score += 2
+    elif opengraph_tags >= 3:
+        score += 1
+
+    # Twitter Cards (3 points)
+    twitter_tags = page.get('twitter_tags', 0)
+    if twitter_tags >= 5:
+        score += 3
+    elif twitter_tags >= 3:
+        score += 2
+    elif twitter_tags >= 1:
+        score += 1
+
+    # ===== CONTENT QUALITY (25 points) =====
+    word_count = page.get('word_count', 0)
+    readability_score = page.get('readability_score', 0)
+    grammar_errors = page.get('grammar_errors', 0)
+    text_to_html_ratio = page.get('text_to_html_ratio', 0)
+    top_keywords = page.get('top_keywords', '')
+
+    # FIXED: Word Count (6 points) - Page-type aware
+    if page_type == 'homepage':
+        # Homepages can be concise
+        if word_count >= 500:
+            score += 6
+        elif word_count >= 300:
+            score += 5
+        elif word_count >= 200:
+            score += 4
+        elif word_count >= 100:
+            score += 3
+        elif word_count >= 50:
+            score += 2
+        else:
+            score += 1
+    elif page_type == 'article':
+        # Articles need depth
+        if word_count >= 2000:
+            score += 6
+        elif word_count >= 1200:
+            score += 5
+        elif word_count >= 800:
+            score += 4
+        elif word_count >= 500:
+            score += 3
+        elif word_count >= 300:
+            score += 2
+        else:
+            score += 1
+    else:
+        # Standard pages
+        if word_count >= 1500:
+            score += 6
+        elif word_count >= 800:
+            score += 5
+        elif word_count >= 500:
+            score += 4
+        elif word_count >= 300:
+            score += 3
+        elif word_count >= 150:
+            score += 2
+        else:
+            score += 1
+
+    # Readability (6 points)
+    if readability_score >= 50:
+        score += 6
+    elif readability_score >= 45:
+        score += 5
+    elif readability_score >= 40:
+        score += 4
+    elif readability_score >= 35:
+        score += 3
+    elif readability_score >= 20:
+        score += 2
+    elif readability_score >= 10:
+        score += 1
+
+    # Grammar (4 points)
+    if grammar_errors == 0:
+        score += 4
+    elif grammar_errors <= 2:
+        score += 3
+    elif grammar_errors <= 5:
+        score += 2
+    elif grammar_errors <= 10:
+        score += 1
+
+    # Text to HTML Ratio (5 points)
+    if text_to_html_ratio >= 25:
+        score += 5
+    elif text_to_html_ratio >= 20:
+        score += 4
+    elif text_to_html_ratio >= 15:
+        score += 3
+    elif text_to_html_ratio >= 10:
+        score += 2
+    elif text_to_html_ratio >= 5:
+        score += 1
+
+    # Keyword Optimization (4 points)
+    if top_keywords:
+        keyword_entries = [k for k in top_keywords.split(', ') if ':' in k and float(k.split(':')[1][:-1]) > 1.0]
+        keyword_count = len(keyword_entries)
+
+        if keyword_count >= 8:
+            score += 4
+        elif keyword_count >= 5:
+            score += 3
+        elif keyword_count >= 3:
+            score += 2
+        elif keyword_count >= 1:
+            score += 1
+
+    return min(score, max_score)
+
+# def calculate_seo_score(page):
+#     """Calculate comprehensive SEO score"""
+#     score = 0
+#     max_score = 100
+
+#     # ===== TITLE OPTIMIZATION (10 points) =====
+#     title = page.get('title', '')
+#     if title:
+#         title_len = len(title)
+#         if 50 <= title_len <= 60:  # Perfect
+#             score += 10
+#         elif 45 <= title_len <= 65:  # Good
+#             score += 8
+#         elif 30 <= title_len <= 70:  # Acceptable
+#             score += 6
+#         elif title_len > 0:  # Exists but poor
+#             score += 3
+
+#     # ===== META DESCRIPTION (8 points) =====
+#     meta_desc = page.get('meta_description', '')
+#     if meta_desc:
+#         meta_len = len(meta_desc)
+#         if 120 <= meta_len <= 155:  # Perfect
+#             score += 8
+#         elif 100 <= meta_len <= 160:  # Good
+#             score += 6
+#         elif 70 <= meta_len <= 170:  # Acceptable
+#             score += 4
+#         elif meta_len > 0:  # Exists but poor
+#             score += 2
+
+#     # ===== HEADING STRUCTURE (12 points) =====
+#     h1_count = page.get('h1_count', 0)
+#     heading_order = page.get('heading_order', '')
+
+#     # H1 Score (6 points)
+#     if h1_count == 1:  # Perfect
+#         score += 6
+#     elif h1_count == 0:  # Critical
+#         score += 0
+#     elif h1_count == 2:  # Minor issue
+#         score += 4
+#     else:  # Multiple H1s
+#         score += 1
+
+#     # Heading Hierarchy (6 points)
+#     if heading_order:
+#         headings = [h.strip() for h in heading_order.split(',')]
+#         heading_levels = []
+
+#         for heading in headings:
+#             if heading.startswith('h'):
+#                 try:
+#                     level = int(heading[1])
+#                     heading_levels.append(level)
+#                 except:
+#                     continue
+
+#         if len(heading_levels) >= 3:
+#             has_h1 = 1 in heading_levels
+#             has_h2 = 2 in heading_levels
+#             has_multiple_levels = len(set(heading_levels)) >= 2
+
+#             if has_h1 and has_h2 and has_multiple_levels:
+#                 score += 6
+#             elif has_h1 and has_multiple_levels:
+#                 score += 4
+#             elif has_h1:
+#                 score += 2
+#         elif len(heading_levels) >= 1:
+#             score += 2
+
+#     # ===== IMAGE OPTIMIZATION (15 points) =====
+#     total_images = page.get('total_images', 0)
+#     missing_alt_tags = page.get('missing_alt_tags', 0)
+#     small_images = page.get('small_images', 0)
+#     large_images = page.get('large_images', 0)
+#     ideal_images = page.get('ideal_images', 0)
+
+#     # Alt Text Score (5 points)
+#     if total_images == 0:
+#         score += 5
+#     else:
+#         alt_ratio = (total_images - missing_alt_tags) / total_images
+#         if alt_ratio >= 0.95:
+#             score += 5
+#         elif alt_ratio >= 0.80:
+#             score += 4
+#         elif alt_ratio >= 0.60:
+#             score += 3
+#         elif alt_ratio >= 0.40:
+#             score += 2
+#         elif alt_ratio > 0:
+#             score += 1
+
+#     # Image Size Optimization (5 points)
+#     if total_images > 0:
+#         ideal_ratio = ideal_images / total_images
+#         if ideal_ratio >= 0.8:
+#             score += 5
+#         elif ideal_ratio >= 0.6:
+#             score += 4
+#         elif ideal_ratio >= 0.4:
+#             score += 3
+#         elif ideal_ratio >= 0.2:
+#             score += 2
+#         else:
+#             score += 1
+#     else:
+#         score += 5
+
+#     # Image Quantity (5 points)
+#     if total_images == 0:
+#         score += 3
+#     elif 3 <= total_images <= 20:
+#         score += 5
+#     elif total_images <= 50:
+#         score += 4
+#     elif total_images <= 100:
+#         score += 3
+#     else:
+#         score += 2
+
+#     # ===== LINK STRUCTURE (10 points) =====
+#     internal_links = page.get('internal_links', 0)
+#     external_links = page.get('external_links', 0)
+
+#     # Internal Links (6 points)
+#     if internal_links >= 80:
+#         score += 6
+#     elif internal_links >= 60:
+#         score += 5
+#     elif internal_links >= 40:
+#         score += 4
+#     elif internal_links >= 20:
+#         score += 3
+#     elif internal_links >= 10:
+#         score += 2
+
+#     # External Links (4 points)
+#     if external_links >= 10:
+#         score += 4
+#     elif external_links >= 7:
+#         score += 3
+#     elif external_links >= 4:
+#         score += 2
+
+#     # ===== TECHNICAL SEO (20 points) =====
+#     # Canonical Tag (3 points)
+#     if page.get('canonical_tag', False):
+#         score += 3
+
+#     # Robots Meta (3 points)
+#     robots_meta = page.get('robots_meta', '')
+#     if robots_meta:
+#         robots_lower = robots_meta.lower()
+#         if 'noindex' not in robots_lower and 'nofollow' not in robots_lower:
+#             score += 3
+#         elif 'noindex' in robots_lower:
+#             score += 0
+#         else:
+#             score += 2
+#     else:
+#         score += 2
+
+#     # Viewport (3 points)
+#     if page.get('viewport_present', False):
+#         score += 3
+
+#     # Schema Markup (4 points)
+#     schema_types = page.get('schema_types', '')
+#     if schema_types and schema_types.strip() and schema_types != "No schema found":
+#         schema_count = len([s for s in schema_types.split(', ') if s.strip()])
+#         if schema_count >= 3:
+#             score += 4
+#         elif schema_count >= 2:
+#             score += 3
+#         elif schema_count == 1:
+#             score += 2
+
+#     # OpenGraph Tags (4 points)
+#     opengraph_tags = page.get('opengraph_tags', 0)
+#     if opengraph_tags >= 10:
+#         score += 4
+#     elif opengraph_tags >= 7:
+#         score += 3
+#     elif opengraph_tags >= 5:
+#         score += 2
+#     elif opengraph_tags >= 3:
+#         score += 1
+
+#     # Twitter Cards (3 points)
+#     twitter_tags = page.get('twitter_tags', 0)
+#     if twitter_tags >= 5:
+#         score += 3
+#     elif twitter_tags >= 3:
+#         score += 2
+#     elif twitter_tags >= 1:
+#         score += 1
+
+#     # ===== CONTENT QUALITY (25 points) =====
+#     word_count = page.get('word_count', 0)
+#     readability_score = page.get('readability_score', 0)
+#     grammar_errors = page.get('grammar_errors', 0)
+#     text_to_html_ratio = page.get('text_to_html_ratio', 0)
+#     top_keywords = page.get('top_keywords', '')
+
+#     # Word Count (6 points)
+#     if word_count >= 2000:
+#         score += 6
+#     elif word_count >= 1200:
+#         score += 5
+#     elif word_count >= 800:
+#         score += 4
+#     elif word_count >= 500:
+#         score += 3
+#     elif word_count >= 300:
+#         score += 2
+#     elif word_count >= 150:
+#         score += 1
+
+#     # Readability (6 points)
+#     if readability_score >= 50:
+#         score += 6
+#     elif readability_score >= 45:
+#         score += 5
+#     elif readability_score >= 40:
+#         score += 4
+#     elif readability_score >= 35:
+#         score += 3
+#     elif readability_score >= 20:
+#         score += 2
+#     elif readability_score >= 10:
+#         score += 1
+
+#     # Grammar (4 points)
+#     if grammar_errors == 0:
+#         score += 4
+#     elif grammar_errors <= 2:
+#         score += 3
+#     elif grammar_errors <= 5:
+#         score += 2
+#     elif grammar_errors <= 10:
+#         score += 1
+
+#     # Text to HTML Ratio (5 points)
+#     if text_to_html_ratio >= 25:
+#         score += 5
+#     elif text_to_html_ratio >= 20:
+#         score += 4
+#     elif text_to_html_ratio >= 15:
+#         score += 3
+#     elif text_to_html_ratio >= 10:
+#         score += 2
+#     elif text_to_html_ratio >= 5:
+#         score += 1
+
+#     # Keyword Optimization (4 points)
+#     if top_keywords:
+#         keyword_entries = [k for k in top_keywords.split(', ') if ':' in k and float(k.split(':')[1][:-1]) > 1.0]
+#         keyword_count = len(keyword_entries)
+
+#         if keyword_count >= 8:
+#             score += 4
+#         elif keyword_count >= 5:
+#             score += 3
+#         elif keyword_count >= 3:
+#             score += 2
+#         elif keyword_count >= 1:
+#             score += 1
+
+#     return min(score, max_score)
+
+# ==============================
+# ENHANCED MAIN FUNCTION WITH ULTRA-SPECIFIC AI SUGGESTIONS
+# ==============================
+async def run_seo_and_suggestions_async(base_url, max_pages=20, tmp_dir="/tmp", use_ai=True, max_concurrent=3):
+    """
+    FULLY PARALLEL SEO analysis with Playwright - ENHANCED WITH ULTRA-SPECIFIC AI SUGGESTIONS
+    """
+    if not base_url:
+        raise ValueError("base_url is required")
+
+    print(f"🎯 Starting ULTRA-SPECIFIC SEO analysis for: {base_url}")
+    print(f"⚡ Optimized: {max_concurrent} concurrent browsers, {max_pages} max pages")
+    print(f"🤖 ULTRA-SPECIFIC AI Suggestions: {'ENABLED' if use_ai and OPENAI_AVAILABLE and os.environ.get('OPENAI_API_KEY') else 'DISABLED'}")
+    start_time = time.time()
+
+    domain = urlparse(base_url).netloc
+
+    try:
+        # STEP 1: Discover URLs in parallel
+        urls = await discover_urls_parallel(base_url, max_pages)
+
+        if not urls:
+            print("❌ No URLs discovered, using homepage only as fallback")
+            urls = [base_url]
+
+        print(f"🔍 Found {len(urls)} URLs to analyze")
+
+        # STEP 2: Fetch all pages in parallel with Playwright
+        playwright_data = await fetch_all_pages_parallel(urls, max_concurrent)
+
+        if not playwright_data:
+            print("❌ No pages could be fetched, creating error report")
+            # Create error report
+            os.makedirs(tmp_dir, exist_ok=True)
+            filename = os.path.join(tmp_dir, f"seo_report_error_{uuid.uuid4().hex}.csv")
+            error_data = [{
+                "url": base_url,
+                "error": "Failed to fetch any pages. Site may be blocking bots or require authentication.",
+                "seo_suggestions": "Check if the site is accessible and not blocking headless browsers."
+            }]
+            with open(filename, "w", newline="", encoding="utf-8") as f:
+                writer = csv.DictWriter(f, fieldnames=error_data[0].keys())
+                writer.writeheader()
+                writer.writerows(error_data)
+            return error_data, filename
+
+        # STEP 3: Analyze all pages in parallel
+        results = await analyze_pages_parallel(playwright_data, domain)
+
+        if not results:
+            print("❌ No pages could be analyzed, creating error report")
+            # Create error report
+            os.makedirs(tmp_dir, exist_ok=True)
+            filename = os.path.join(tmp_dir, f"seo_report_error_{uuid.uuid4().hex}.csv")
+            error_data = [{
+                "url": base_url,
+                "error": "Failed to analyze any pages. There may be issues with the page content.",
+                "seo_suggestions": "Check if the site has proper HTML structure and content."
+            }]
+            with open(filename, "w", newline="", encoding="utf-8") as f:
+                writer = csv.DictWriter(f, fieldnames=error_data[0].keys())
+                writer.writeheader()
+                writer.writerows(error_data)
+            return error_data, filename
+
+        # STEP 4: Calculate SEO scores
+        print("📊 Calculating SEO scores...")
+        for p in results:
+            p["seo_score"] = calculate_seo_score(p)
+
+        # STEP 5: Generate ULTRA-SPECIFIC AI suggestions in parallel
+        if use_ai:
+            await add_comprehensive_suggestions_async(results)
+        else:
+            for p in results:
+                p["ai_suggestions"] = "AI suggestions disabled - set use_ai=True and OPENAI_API_KEY"
+
+        # STEP 6: Save to CSV
+        os.makedirs(tmp_dir, exist_ok=True)
+        filename = os.path.join(tmp_dir, f"seo_report_ultra_specific_{uuid.uuid4().hex}.csv")
+
+        if results:
+            keys = list(results[0].keys())
+            with open(filename, "w", newline="", encoding="utf-8") as f:
+                writer = csv.DictWriter(f, fieldnames=keys)
+                writer.writeheader()
+                writer.writerows(results)
+
+        elapsed_time = time.time() - start_time
+        print(f"✅ ULTRA-SPECIFIC SEO analysis complete! Analyzed {len(results)} pages in {elapsed_time:.1f}s")
+        print(f"📊 Report saved to: {filename}")
+
+        # Count ULTRA-SPECIFIC AI suggestions
+        ultra_specific_count = sum(1 for p in results if p.get('ai_suggestions') and
+                                  'AI disabled' not in p.get('ai_suggestions', '') and
+                                  'AI Error' not in p.get('ai_suggestions', ''))
+        print(f"🤖 ULTRA-SPECIFIC AI Suggestions: {ultra_specific_count}/{len(results)} pages")
+
+        return results, filename
+
+    except Exception as e:
+        print(f"❌ Unexpected error during SEO analysis: {e}")
+        # Create error report
+        os.makedirs(tmp_dir, exist_ok=True)
+        filename = os.path.join(tmp_dir, f"seo_report_error_{uuid.uuid4().hex}.csv")
+        error_data = [{
+            "url": base_url,
+            "error": f"Unexpected error: {str(e)}",
+            "seo_suggestions": "Please check the website URL and try again."
+        }]
+        with open(filename, "w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=error_data[0].keys())
+            writer.writeheader()
+            writer.writerows(error_data)
+        return error_data, filename
+
+# ==============================
+# ASYNC WRAPPER FOR FASTAPI
+# ==============================
+async def run_seo_analysis_fastapi(base_url, max_pages=5, use_ai=True, max_concurrent=2, download=False):
+    """
+    Async wrapper for FastAPI compatibility
+    """
+    try:
+        results, csv_path = await run_seo_and_suggestions_async(
+            base_url=base_url,
+            max_pages=max_pages,
+            tmp_dir="/tmp",
+            use_ai=use_ai,
+            max_concurrent=max_concurrent
+        )
+        return results, csv_path
+    except Exception as e:
+        filename = f"/tmp/seo_report_error_{uuid.uuid4().hex}.csv"
+        error_data = [{
+            "url": base_url,
+            "error": f"Analysis failed: {str(e)}",
+            "seo_suggestions": "Please check the website URL and try again."
+        }]
+        with open(filename, "w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=error_data[0].keys())
+            writer.writeheader()
+            writer.writerows(error_data)
+        return error_data, filename