Update ai_visibility.py

ai_visibility.py

@@ -1,3 +1,6 @@
+
+
+
 """
 AI Visibility / AI Search Readiness analysis.
 
@@ -26,6 +29,38 @@ It reuses the same crawling primitives (discover_urls_parallel /
 fetch_all_pages_parallel) so pages are only fetched once per run via
 Playwright, but it does its own HTML parsing and its own scoring - it never
 touches or overrides the existing seo_score / page_summary fields.
+
+============================================================================
+CHANGELOG (bugfixes applied in this revision)
+============================================================================
+1. FRESHNESS DATE PARSING (the main reported bug): `_analyze_freshness`
+   correctly *detected* dates in formats like "November 5, 2024" or
+   "5 November 2024" via DATE_PATTERNS, but the parsing step only tried
+   strict ISO formats ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"). Any date that
+   wasn't already ISO-formatted failed to parse, so `content_age_days`
+   stayed None and `freshness_status` silently fell through to "unknown"
+   even though a date was clearly visible on the page. This made the
+   "freshness" score None/0 on the vast majority of real-world sites,
+   since most CMSs render human-readable dates, not ISO strings.
+   FIX: added a broad list of strptime formats plus a dateutil fallback
+   (fuzzy=True) so any commonly-rendered date string parses correctly.
+
+2. ANSWER COVERAGE UNDER-DETECTION: `_analyze_answerability` only counted
+   a heading as "question-style" if its text ended with a literal "?".
+   Real FAQ/help content very often uses headings like "How to reset your
+   password" with no question mark, so `question_count` stayed 0 and
+   `answer_coverage` was reported as None (shown as 0 downstream) even on
+   pages with clear Q&A structure.
+   FIX: headings are now also matched if they start with a common question
+   word/phrase (how/what/why/when/where/can/do/does/is/are/should/will),
+   in addition to the original "ends with ?" check.
+
+3. Restored the truncated end of the file (the `unknown_metrics` key in
+   the final returned dict) - the last document pasted in this
+   conversation was cut off mid-line.
+
+No other scoring logic, weights, or public behavior was changed.
+============================================================================
 """
 
 import os
@@ -40,6 +75,11 @@ load_dotenv()
 
 from bs4 import BeautifulSoup
 
+try:
+    from dateutil import parser as _dateutil_parser
+except Exception:
+    _dateutil_parser = None
+
 from seo_analyzer import (
     discover_urls_parallel,
     fetch_all_pages_parallel,
@@ -198,6 +238,15 @@ DATE_PATTERNS = [
     re.compile(r'\b\d{1,2}\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+(19|20)\d{2}\b', re.I),
 ]
 
+# FIX (#2): question headings previously only matched if the text ended in
+# a literal "?". Real-world FAQ/help headings are frequently phrased as
+# imperative/declarative "questions" without a trailing "?" (e.g. "How to
+# reset your password"). This pattern catches those too.
+QUESTION_START_PATTERN = re.compile(
+    r'^(how|what|why|when|where|who|which|can|could|do|does|did|is|are|was|were|should|will|would)\b',
+    re.I,
+)
+
 FIRST_HAND_PATTERNS = re.compile(
     r'\b(we tested|we found|our (research|study|testing|analysis|experiment)|in our experience|hands[- ]on|i tested|i used|we measured|we surveyed)\b',
     re.I,
@@ -220,6 +269,19 @@ def _clamp(v, lo=0, hi=100):
     return max(lo, min(hi, v))
 
 
+def _is_question_heading(heading_text):
+    """FIX (#2): a heading counts as 'question-style' if it ends with '?'
+    OR starts with a common question word/phrase. Broadens detection beyond
+    the original ends-with-'?' -only check so real FAQ/help headings that
+    omit the question mark are still picked up."""
+    t = (heading_text or "").strip()
+    if not t:
+        return False
+    if t.endswith("?"):
+        return True
+    return bool(QUESTION_START_PATTERN.match(t))
+
+
 # ==============================
 # PAGE TYPE CLASSIFICATION (deterministic)
 # ==============================
@@ -233,7 +295,7 @@ def classify_page_type(seo_data, soup, text, word_count, schema_types, links):
     lower_types = [t.lower() for t in schema_types]
 
     heading_tags = soup.find_all(re.compile("^h[1-6]$"))
-    question_headings = sum(1 for h in heading_tags if h.get_text(strip=True).endswith("?"))
+    question_headings = sum(1 for h in heading_tags if _is_question_heading(h.get_text(strip=True)))
     breadcrumb_present = bool(soup.find(attrs={"class": re.compile("breadcrumb", re.I)})) or any(
         "breadcrumblist" in t for t in lower_types
     )
@@ -506,7 +568,9 @@ def _analyze_entities(seo_data, soup, text, word_count, schema_types, page_type)
 # ==============================
 def _analyze_answerability(soup, text):
     heading_tags = soup.find_all(re.compile("^h[1-6]$"))
-    question_headings = [h for h in heading_tags if h.get_text(strip=True).endswith("?")]
+    # FIX (#2): use the broadened _is_question_heading check instead of the
+    # original "ends with literal '?'" -only test.
+    question_headings = [h for h in heading_tags if _is_question_heading(h.get_text(strip=True))]
     question_count = len(question_headings)
 
     def _next_text_len(tag):
@@ -852,6 +916,61 @@ def _analyze_citation_potential(soup, text):
 # CATEGORY 10: FRESHNESS (fresh / stale / very_stale / unknown - never
 # "guessed outdated")
 # ==============================
+def _parse_flexible_date(raw_date):
+    """FIX (#1 - the reported bug): the original implementation only tried
+    two strict ISO strptime formats, so any human-readable date detected by
+    DATE_PATTERNS (e.g. "November 5, 2024", "5 November 2024") failed to
+    parse and silently produced freshness_status == 'unknown' on almost
+    every real page. This now tries a broad set of explicit formats first
+    (fast, no dependency needed) and falls back to dateutil's fuzzy parser
+    for anything else. Returns a timezone-aware UTC datetime, or None.
+    """
+    if not raw_date:
+        return None
+
+    from datetime import datetime, timezone
+
+    candidate = raw_date.strip()
+
+    explicit_formats = (
+        "%Y-%m-%dT%H:%M:%S%z",
+        "%Y-%m-%dT%H:%M:%S",
+        "%Y-%m-%d",
+        "%B %d, %Y",   # November 5, 2024
+        "%B %d %Y",    # November 5 2024
+        "%d %B %Y",    # 5 November 2024
+        "%b %d, %Y",   # Nov 5, 2024
+        "%b %d %Y",    # Nov 5 2024
+        "%d %b %Y",    # 5 Nov 2024
+        "%m/%d/%Y",
+        "%d/%m/%Y",
+    )
+
+    # Normalize a trailing 'Z' (UTC designator) so the '%z'-less formats work.
+    normalized = candidate[:19].replace("Z", "") if len(candidate) >= 19 else candidate
+
+    for fmt in explicit_formats:
+        for attempt in (candidate, normalized):
+            try:
+                parsed = datetime.strptime(attempt, fmt)
+                if parsed.tzinfo is None:
+                    parsed = parsed.replace(tzinfo=timezone.utc)
+                return parsed
+            except Exception:
+                continue
+
+    if _dateutil_parser is not None:
+        try:
+            parsed = _dateutil_parser.parse(candidate, fuzzy=True)
+            if parsed.tzinfo is None:
+                parsed = parsed.replace(tzinfo=timezone.utc)
+            return parsed
+        except Exception:
+            return None
+
+    return None
+
+
 def _analyze_freshness(seo_data, soup, text, page_type):
     metas = seo_data.get("metas", {}) or {}
     last_updated = (
@@ -878,15 +997,8 @@ def _analyze_freshness(seo_data, soup, text, page_type):
     if last_updated:
         try:
             from datetime import datetime, timezone
-            parsed = None
-            for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
-                try:
-                    parsed = datetime.strptime(last_updated[:19].replace("Z", ""), fmt)
-                    break
-                except Exception:
-                    continue
+            parsed = _parse_flexible_date(last_updated)
             if parsed:
-                parsed = parsed.replace(tzinfo=timezone.utc)
                 content_age_days = (datetime.now(timezone.utc) - parsed).days
         except Exception:
             content_age_days = None
@@ -1460,4 +1572,1474 @@ async def run_ai_visibility_analysis(base_url, max_pages=5, max_concurrent=1, us
         "strengths": strengths,
         "unknown_metrics": unknowns,
         "results_preview": pages_summary,
-    }
+    }
+
+
+
+
+
+
+
+# """
+# AI Visibility / AI Search Readiness analysis.
+
+# Estimates how well a page is structured to be understood, retrieved, cited,
+# summarized and recommended by AI-powered search systems (ChatGPT, Google AI
+# Overviews, Perplexity, Claude, etc). These are NOT official ranking factors of
+# any AI system - they are deterministic, measurable proxy signals derived from
+# the page's HTML/text plus (optionally) a lightweight LLM pass for the handful
+# of metrics that cannot be reliably computed with parsing alone.
+
+# This module reports an "AI Readiness Score" - whether a page is technically
+# and semantically prepared for AI systems. It does NOT claim to measure actual
+# observed visibility in AI answers/citations (query coverage, citation rate,
+# etc) - that requires live AI-query data this crawler does not have, so that
+# field is always returned as "not_measured" rather than guessed at.
+
+# Key design principles (see README/PR notes for the full rationale):
+#   - Pages are classified by type (homepage, article, product, ...) and only
+#     scored against metrics that are relevant to that type.
+#   - Missing evidence is reported as "unknown" / null, never coerced to a low
+#     score. Unknown != bad.
+#   - Weights are centralized and configurable, per page type.
+
+# This module is intentionally decoupled from seo_analyzer.py's SEO scoring.
+# It reuses the same crawling primitives (discover_urls_parallel /
+# fetch_all_pages_parallel) so pages are only fetched once per run via
+# Playwright, but it does its own HTML parsing and its own scoring - it never
+# touches or overrides the existing seo_score / page_summary fields.
+# """
+
+# import os
+# import re
+# import json
+# import asyncio
+# from collections import Counter
+# from urllib.parse import urlparse
+# from urllib.request import urlopen, Request
+# from dotenv import load_dotenv
+# load_dotenv()
+
+# from bs4 import BeautifulSoup
+
+# from seo_analyzer import (
+#     discover_urls_parallel,
+#     fetch_all_pages_parallel,
+#     OPENAI_AVAILABLE,
+# )
+
+# try:
+#     import openai
+# except Exception:
+#     openai = None
+
+# # SECURITY FIX: never hardcode API keys in source. Load from environment.
+# # Set this in your shell / deployment config, e.g.:
+# #   export OPENAI_API_KEY="sk-..."
+# # If it's unset, `use_ai=True` calls will simply fall back to the
+# # deterministic-only scoring path (see _llm_semantic_enhance below).
+# OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+# # Reused across calls so we're not re-instantiating the client every request.
+# _openai_client = None
+
+
+# def _get_openai_client():
+#     global _openai_client
+#     if _openai_client is None and openai is not None and OPENAI_API_KEY:
+#         _openai_client = openai.OpenAI(api_key=OPENAI_API_KEY)
+#     return _openai_client
+
+
+# # ==============================
+# # PAGE TYPES
+# # ==============================
+# PAGE_TYPES = [
+#     "homepage", "portal", "article", "blog_post", "news", "product",
+#     "product_category", "documentation", "forum", "review", "comparison",
+#     "organization", "landing_page", "service_page", "directory",
+#     "search_page", "unknown",
+# ]
+
+# # ==============================
+# # CENTRALIZED SCORING CONFIGURATION
+# # ==============================
+# # Base ("default") weights - used for page types without a specific profile.
+# # Categories are the 8 "AI Readiness" pillars. Weights sum to 1.0.
+# DEFAULT_WEIGHTS = {
+#     "semantic": 0.20,
+#     "content_answerability": 0.25,
+#     "entity": 0.15,
+#     "trust": 0.15,
+#     "citation": 0.10,
+#     "retrieval": 0.05,
+#     "structured_data": 0.05,
+#     "freshness": 0.05,
+# }
+
+# # Per-page-type overrides. Only categories that differ from DEFAULT_WEIGHTS
+# # need to be listed - the profile is merged over the default and renormalized.
+# PAGE_TYPE_WEIGHT_OVERRIDES = {
+#     "homepage": {
+#         "semantic": 0.15, "content_answerability": 0.10, "entity": 0.25,
+#         "trust": 0.15, "citation": 0.05, "retrieval": 0.20,
+#         "structured_data": 0.05, "freshness": 0.05,
+#     },
+#     "portal": {
+#         "semantic": 0.15, "content_answerability": 0.10, "entity": 0.25,
+#         "trust": 0.15, "citation": 0.05, "retrieval": 0.20,
+#         "structured_data": 0.05, "freshness": 0.05,
+#     },
+#     "article": {
+#         "semantic": 0.20, "content_answerability": 0.25, "entity": 0.10,
+#         "trust": 0.15, "citation": 0.15, "retrieval": 0.05,
+#         "structured_data": 0.05, "freshness": 0.05,
+#     },
+#     "blog_post": {
+#         "semantic": 0.20, "content_answerability": 0.25, "entity": 0.10,
+#         "trust": 0.15, "citation": 0.15, "retrieval": 0.05,
+#         "structured_data": 0.05, "freshness": 0.05,
+#     },
+#     "news": {
+#         "semantic": 0.18, "content_answerability": 0.22, "entity": 0.12,
+#         "trust": 0.15, "citation": 0.13, "retrieval": 0.05,
+#         "structured_data": 0.05, "freshness": 0.10,
+#     },
+#     "documentation": {
+#         "semantic": 0.15, "content_answerability": 0.30, "entity": 0.10,
+#         "trust": 0.10, "citation": 0.10, "retrieval": 0.10,
+#         "structured_data": 0.10, "freshness": 0.05,
+#     },
+#     "product": {
+#         "semantic": 0.10, "content_answerability": 0.20, "entity": 0.15,
+#         "trust": 0.10, "citation": 0.05, "retrieval": 0.10,
+#         "structured_data": 0.25, "freshness": 0.05,
+#     },
+#     "product_category": {
+#         "semantic": 0.10, "content_answerability": 0.15, "entity": 0.15,
+#         "trust": 0.10, "citation": 0.05, "retrieval": 0.15,
+#         "structured_data": 0.25, "freshness": 0.05,
+#     },
+#     "forum": {
+#         "semantic": 0.15, "content_answerability": 0.20, "entity": 0.10,
+#         "trust": 0.15, "citation": 0.15, "retrieval": 0.10,
+#         "structured_data": 0.05, "freshness": 0.10,
+#     },
+#     "review": {
+#         "semantic": 0.15, "content_answerability": 0.20, "entity": 0.10,
+#         "trust": 0.20, "citation": 0.15, "retrieval": 0.05,
+#         "structured_data": 0.05, "freshness": 0.10,
+#     },
+#     "comparison": {
+#         "semantic": 0.18, "content_answerability": 0.22, "entity": 0.12,
+#         "trust": 0.15, "citation": 0.13, "retrieval": 0.05,
+#         "structured_data": 0.05, "freshness": 0.10,
+#     },
+#     "organization": {
+#         "semantic": 0.15, "content_answerability": 0.10, "entity": 0.25,
+#         "trust": 0.25, "citation": 0.05, "retrieval": 0.10,
+#         "structured_data": 0.05, "freshness": 0.05,
+#     },
+#     "service_page": {
+#         "semantic": 0.15, "content_answerability": 0.15, "entity": 0.20,
+#         "trust": 0.20, "citation": 0.05, "retrieval": 0.10,
+#         "structured_data": 0.10, "freshness": 0.05,
+#     },
+#     "landing_page": {
+#         "semantic": 0.15, "content_answerability": 0.10, "entity": 0.20,
+#         "trust": 0.20, "citation": 0.05, "retrieval": 0.15,
+#         "structured_data": 0.10, "freshness": 0.05,
+#     },
+# }
+
+
+# def get_weights_for_page_type(page_type):
+#     profile = dict(DEFAULT_WEIGHTS)
+#     profile.update(PAGE_TYPE_WEIGHT_OVERRIDES.get(page_type, {}))
+#     total = sum(profile.values()) or 1.0
+#     return {k: v / total for k, v in profile.items()}
+
+
+# # Per-page-type relevance map: which "trust/answerability" sub-signals are
+# # actually meaningful for this page type. Signals not listed are treated as
+# # not_applicable (excluded from scoring) rather than penalized when absent.
+# TYPES_WHERE_AUTHOR_RELEVANT = {
+#     "article", "blog_post", "news", "review", "comparison", "documentation", "forum",
+# }
+# TYPES_WHERE_FRESHNESS_RELEVANT = {
+#     "article", "blog_post", "news", "review", "comparison", "documentation", "forum", "product",
+# }
+# TYPES_WHERE_FAQ_RELEVANT = {
+#     "article", "blog_post", "documentation", "product", "product_category", "service_page", "landing_page",
+# }
+# TYPES_WHERE_PRODUCT_SCHEMA_RELEVANT = {"product", "product_category"}
+# TYPES_WHERE_ARTICLE_SCHEMA_RELEVANT = {"article", "blog_post", "news"}
+
+# DATE_PATTERNS = [
+#     re.compile(r'\b(19|20)\d{2}-\d{2}-\d{2}\b'),
+#     re.compile(r'\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+(19|20)\d{2}\b', re.I),
+#     re.compile(r'\b\d{1,2}\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+(19|20)\d{2}\b', re.I),
+# ]
+
+# FIRST_HAND_PATTERNS = re.compile(
+#     r'\b(we tested|we found|our (research|study|testing|analysis|experiment)|in our experience|hands[- ]on|i tested|i used|we measured|we surveyed)\b',
+#     re.I,
+# )
+
+# SOURCE_ATTRIBUTION_PATTERNS = re.compile(
+#     r'\b(according to|source:|cited by|reported by|study by|research (from|by)|as reported)\b', re.I
+# )
+
+
+# def _words(text):
+#     return re.findall(r"[A-Za-z']+", text or "")
+
+
+# def _pct(n, d):
+#     return round((n / d) * 100, 1) if d else 0.0
+
+
+# def _clamp(v, lo=0, hi=100):
+#     return max(lo, min(hi, v))
+
+
+# # ==============================
+# # PAGE TYPE CLASSIFICATION (deterministic)
+# # ==============================
+# def classify_page_type(seo_data, soup, text, word_count, schema_types, links):
+#     """Classify the page using URL structure, schema, and content-structure
+#     signals. Returns {"type": str, "confidence": float, "signals": [...]}.
+#     No LLM call - this must be fast and run for every page.
+#     """
+#     url = seo_data.get("url", "")
+#     path = urlparse(url).path.strip("/").lower()
+#     lower_types = [t.lower() for t in schema_types]
+
+#     heading_tags = soup.find_all(re.compile("^h[1-6]$"))
+#     question_headings = sum(1 for h in heading_tags if h.get_text(strip=True).endswith("?"))
+#     breadcrumb_present = bool(soup.find(attrs={"class": re.compile("breadcrumb", re.I)})) or any(
+#         "breadcrumblist" in t for t in lower_types
+#     )
+#     has_price = bool(re.search(r'(\$|USD|EUR|£|₹)\s?\d', text or ""))
+#     add_to_cart = bool(re.search(r'add to cart|buy now|add to bag|add to trolley', text or "", re.I))
+#     nav_link_count = len(links)
+
+#     votes = Counter()
+#     signals = []
+
+#     def vote(t, n, reason):
+#         votes[t] += n
+#         signals.append(f"{t}+{n}:{reason}")
+
+#     # --- URL structure signals ---
+#     if path == "":
+#         vote("homepage", 3, "root path")
+#     if re.search(r'\b(blog|article|post)\b', path):
+#         vote("blog_post", 2, "url path")
+#     if re.search(r'\bnews\b', path):
+#         vote("news", 2, "url path")
+#     if re.search(r'\b(docs?|documentation|guide|help|kb|support|wiki)\b', path):
+#         vote("documentation", 2, "url path")
+#     if re.search(r'\b(product|item|shop|store)\b', path):
+#         vote("product", 2, "url path")
+#     if re.search(r'\b(category|collection|catalog|categories)\b', path):
+#         vote("product_category", 2, "url path")
+#     if re.search(r'\b(forum|thread|topic|community|discussion)\b', path):
+#         vote("forum", 2, "url path")
+#     if re.search(r'\breview', path):
+#         vote("review", 2, "url path")
+#     if re.search(r'\b(vs|compare|comparison|alternatives)\b', path):
+#         vote("comparison", 2, "url path")
+#     if re.search(r'\b(about|company|who-we-are|team)\b', path):
+#         vote("organization", 2, "url path")
+#     if re.search(r'\b(pricing|services|solutions|features)\b', path):
+#         vote("service_page", 1, "url path")
+#     if re.search(r'\bsearch\b', path) or "q=" in urlparse(url).query:
+#         vote("search_page", 2, "url/query")
+#     if re.search(r'\b(directory|listings?)\b', path):
+#         vote("directory", 1, "url path")
+#     if re.search(r'\b(landing|lp)\b', path):
+#         vote("landing_page", 1, "url path")
+
+#     # --- schema.org signals ---
+#     if "product" in lower_types:
+#         vote("product", 3, "Product schema")
+#     if any(t in lower_types for t in ("article", "newsarticle", "blogposting")):
+#         vote("article" if "newsarticle" not in lower_types else "news", 3, "Article-family schema")
+#     if "organization" in lower_types and path == "":
+#         vote("organization", 1, "Organization schema on root")
+#     if "faqpage" in lower_types:
+#         vote("documentation", 1, "FAQPage schema")
+#     if any(t in lower_types for t in ("itemlist", "collectionpage")):
+#         vote("portal", 2, "ItemList/CollectionPage schema")
+#     if "webpage" in lower_types and path == "":
+#         vote("homepage", 1, "WebPage schema on root")
+
+#     # --- structural/content signals ---
+#     if nav_link_count >= 40 and word_count < 800:
+#         vote("portal", 2, "high link density, low unique text")
+#     if path == "" and nav_link_count >= 25:
+#         vote("homepage", 2, "root path with heavy navigation")
+#     if word_count >= 500 and heading_tags and not has_price:
+#         vote("article", 1, "substantial prose content")
+#     if has_price and add_to_cart:
+#         vote("product", 3, "price + add-to-cart")
+#     elif has_price:
+#         vote("product", 1, "price present")
+#     if question_headings >= 3:
+#         vote("documentation", 1, "multiple question-style headings")
+#         vote("forum", 1, "multiple question-style headings")
+#     if breadcrumb_present and word_count > 300:
+#         vote("article", 1, "breadcrumb + substantial content")
+
+#     if not votes:
+#         return {"type": "unknown", "confidence": 0.3, "signals": []}
+
+#     page_type, top_votes = votes.most_common(1)[0]
+#     total_votes = sum(votes.values())
+#     confidence = round(min(0.98, 0.35 + (top_votes / max(1, total_votes)) * 0.6), 2)
+#     return {"type": page_type, "confidence": confidence, "signals": signals}
+
+
+# # ==============================
+# # CATEGORY 1: TOPIC & SEMANTIC UNDERSTANDING (query-independent by default)
+# # ==============================
+# def _analyze_semantic(seo_data, soup, text, word_count, target_query=None):
+#     title = (seo_data.get("title") or "").strip()
+#     description = (seo_data.get("description") or "").strip()
+#     h1s = [h.get_text(" ", strip=True) for h in soup.find_all("h1")]
+#     h1_text = " ".join(h1s)
+
+#     title_words = set(w.lower() for w in _words(title) if len(w) > 3)
+#     h1_words = set(w.lower() for w in _words(h1_text) if len(w) > 3)
+#     desc_words = set(w.lower() for w in _words(description) if len(w) > 3)
+
+#     # topic_clarity: title exists, has an h1, and they share vocabulary.
+#     # This does NOT require an external reference query.
+#     topic_clarity = 0
+#     if title:
+#         topic_clarity += 40
+#     if h1_text:
+#         topic_clarity += 30
+#     if title_words and h1_words:
+#         overlap = len(title_words & h1_words) / max(1, len(title_words | h1_words))
+#         topic_clarity += round(overlap * 30)
+#     topic_clarity = _clamp(topic_clarity)
+
+#     # topic_coherence: does title/description/h1 vocabulary agree with itself
+#     # (internal consistency proxy, not "relevance" to any external query).
+#     all_pairs = [p for p in [
+#         (title_words, desc_words),
+#         (title_words, h1_words),
+#         (desc_words, h1_words),
+#     ] if p[0] and p[1]]
+#     if all_pairs:
+#         sims = [len(a & b) / max(1, len(a | b)) for a, b in all_pairs]
+#         topic_coherence = _clamp(round(sum(sims) / len(sims) * 100))
+#     else:
+#         topic_coherence = 0
+
+#     # content_completeness: word count + heading coverage + list/table presence
+#     heading_count = len(soup.find_all(re.compile("^h[1-6]$")))
+#     completeness = 0
+#     if word_count >= 1000:
+#         completeness += 40
+#     elif word_count >= 500:
+#         completeness += 30
+#     elif word_count >= 300:
+#         completeness += 15
+#     if heading_count >= 3:
+#         completeness += 30
+#     elif heading_count >= 1:
+#         completeness += 15
+#     if soup.find_all(["ul", "ol", "table"]):
+#         completeness += 15
+#     if description:
+#         completeness += 15
+#     content_completeness = _clamp(completeness)
+
+#     # content_depth: vocabulary richness + paragraph count
+#     paragraphs = [p.get_text(" ", strip=True) for p in soup.find_all("p")]
+#     paragraphs = [p for p in paragraphs if p]
+#     unique_words = set(w.lower() for w in _words(text))
+#     richness = _pct(len(unique_words), max(1, word_count))
+#     depth = 0
+#     depth += min(40, round(richness))
+#     depth += min(30, len(paragraphs) * 2)
+#     depth += min(30, heading_count * 5)
+#     content_depth = _clamp(depth)
+
+#     result = {
+#         "topic_clarity": topic_clarity,
+#         "topic_coherence": topic_coherence,
+#         "content_completeness": content_completeness,
+#         "content_depth": content_depth,
+#         "semantic_relevance": None,
+#         "semantic_relevance_status": "requires_target_query",
+#         "search_intent_match": None,
+#         "search_intent_match_status": "requires_target_query",
+#     }
+
+#     # These two metrics genuinely require a reference query/keyword to mean
+#     # anything. Without one they are reported as unknown, not guessed at.
+#     if target_query:
+#         query_words = set(w.lower() for w in _words(target_query) if len(w) > 2)
+#         if query_words:
+#             corpus_pairs = [
+#                 (query_words, title_words),
+#                 (query_words, h1_words),
+#                 (query_words, set(w.lower() for w in _words(text)[:400])),
+#             ]
+#             sims = [len(a & b) / max(1, len(a)) for a, b in corpus_pairs if a]
+#             semantic_relevance = _clamp(round((sum(sims) / len(sims)) * 100)) if sims else 0
+#             result["semantic_relevance"] = semantic_relevance
+#             result["semantic_relevance_status"] = "measured"
+
+#             intent_markers = ["how to", "what is", "why", "best", "guide", "review", "vs", "top ", "buy", "price"]
+#             lowered_query = target_query.lower()
+#             matched_intent = next((m for m in intent_markers if m in lowered_query), None)
+#             overlap_in_content = len(query_words & set(w.lower() for w in _words(text))) / max(1, len(query_words))
+#             search_intent_match = _clamp(round(overlap_in_content * 100))
+#             if matched_intent in ("how to", "guide") and soup.find_all(["ol", "ul"]):
+#                 search_intent_match = _clamp(search_intent_match + 15)
+#             result["search_intent_match"] = search_intent_match
+#             result["search_intent_match_status"] = "measured"
+
+#     return result
+
+
+# # ==============================
+# # CATEGORY 2: ENTITY UNDERSTANDING
+# # ==============================
+# def _analyze_entities(seo_data, soup, text, word_count, schema_types, page_type):
+#     # Proper-noun style heuristic: capitalized word sequences not at sentence start.
+#     # This works from visible text/headings/title/links - it does NOT require
+#     # JSON-LD to detect entities. Schema (below) only boosts confidence.
+#     proper_noun_seqs = re.findall(r'(?<!\. )(?<!^)\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3})\b', text or "")
+#     title_seqs = re.findall(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3})\b', seo_data.get("title") or "")
+#     heading_seqs = []
+#     for h in soup.find_all(re.compile("^h[1-6]$")):
+#         heading_seqs.extend(re.findall(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3})\b', h.get_text(" ", strip=True)))
+
+#     entity_counter = Counter(s.strip() for s in proper_noun_seqs + title_seqs + heading_seqs if len(s.strip()) > 2)
+#     entity_count = len(entity_counter)
+
+#     if word_count < 20:
+#         entity_clarity = None
+#         entity_clarity_status = "insufficient_text"
+#     else:
+#         entity_clarity = _clamp(round(_pct(entity_count, max(1, word_count // 20))))
+#         entity_clarity_status = "measured"
+
+#     if entity_counter:
+#         top_mentions = entity_counter.most_common(1)[0][1]
+#         entity_consistency = _clamp(round(_pct(top_mentions, sum(entity_counter.values()))))
+#     else:
+#         entity_consistency = 0
+
+#     lower_types = [t.lower() for t in schema_types]
+#     organization_entity_present = bool(
+#         {"organization", "corporation", "localbusiness"} & set(lower_types)
+#     ) or bool(soup.find(attrs={"itemtype": re.compile("Organization", re.I)})) or bool(
+#         soup.find("footer") and re.search(r'\b(Inc\.|LLC|Ltd\.|Corporation|Corp\.)\b', text or "")
+#     )
+
+#     author_entity_present = bool(
+#         seo_data.get("metas", {}).get("author")
+#     ) or bool(soup.find(attrs={"rel": "author"})) or bool(
+#         soup.find(class_=re.compile("author|byline", re.I))
+#     ) or "person" in lower_types
+
+#     product_entity_present = "product" in lower_types or bool(
+#         re.search(r'\b(SKU|model number|add to cart)\b', text or "", re.I)
+#     )
+#     place_entity_present = bool(re.search(r'\b\d{5}(-\d{4})?\b', text or "")) and bool(
+#         re.search(r'\b(Street|Ave|Avenue|Road|Blvd|City|Country)\b', text or "", re.I)
+#     )
+#     brand_entity_present = organization_entity_present or bool(
+#         re.search(r'\b(®|™)\b', text or "")
+#     )
+
+#     entity_types_present = [
+#         t for t, present in [
+#             ("organization", organization_entity_present),
+#             ("person", author_entity_present),
+#             ("product", product_entity_present),
+#             ("place", place_entity_present),
+#             ("brand", brand_entity_present),
+#         ] if present
+#     ]
+
+#     return {
+#         "entity_count": entity_count,
+#         "entity_clarity": entity_clarity,
+#         "entity_clarity_status": entity_clarity_status,
+#         "entity_consistency": entity_consistency,
+#         "entity_types": entity_types_present,
+#         "organization_entity_present": organization_entity_present,
+#         "author_entity_present": author_entity_present,
+#         "product_entity_present": product_entity_present,
+#         "place_entity_present": place_entity_present,
+#         "brand_entity_present": brand_entity_present,
+#     }
+
+
+# # ==============================
+# # CATEGORY 3: ANSWERABILITY
+# # ==============================
+# def _analyze_answerability(soup, text):
+#     heading_tags = soup.find_all(re.compile("^h[1-6]$"))
+#     question_headings = [h for h in heading_tags if h.get_text(strip=True).endswith("?")]
+#     question_count = len(question_headings)
+
+#     def _next_text_len(tag):
+#         sib = tag.find_next_sibling()
+#         hops = 0
+#         while sib is not None and hops < 3:
+#             content = sib.get_text(" ", strip=True)
+#             if content:
+#                 return len(_words(content))
+#             sib = sib.find_next_sibling()
+#             hops += 1
+#         return 0
+
+#     answered = sum(1 for h in question_headings if _next_text_len(h) >= 15)
+#     questions_answered = answered
+#     answer_coverage = _pct(answered, question_count) if question_count else None
+
+#     first_p = soup.find("p")
+#     first_p_text = first_p.get_text(" ", strip=True) if first_p else ""
+#     direct_answer_presence = bool(first_p_text) and 40 <= len(first_p_text) <= 400
+
+#     definition_presence = bool(
+#         re.search(r'\b\w+\s+(is|are|refers to|means)\s+(a|an|the)\b', text or "", re.I)
+#     ) or bool(soup.find("dfn")) or bool(soup.find("dl"))
+
+#     faq_heading = soup.find(
+#         lambda t: t.name in ("h1", "h2", "h3") and re.search(r"faq|frequently asked", t.get_text(" ", strip=True), re.I)
+#     )
+#     faq_coverage = 100 if faq_heading else (_clamp(question_count * 20) if question_count else 0)
+
+#     return {
+#         "question_count": question_count,
+#         "questions_answered": questions_answered,
+#         "answer_coverage": answer_coverage,
+#         "direct_answer_presence": direct_answer_presence,
+#         "definition_presence": definition_presence,
+#         "faq_coverage": faq_coverage,
+#     }
+
+
+# # ==============================
+# # CATEGORY 4: INFORMATION QUALITY
+# # ==============================
+# def _analyze_information_quality(soup, text):
+#     sentences = re.split(r'(?<=[.!?])\s+', text or "")
+#     stat_pattern = re.compile(r'\b\d+([.,]\d+)?\s?(%|percent)?\b')
+#     factual_sentences = [s for s in sentences if re.search(r'\d', s) and stat_pattern.search(s)]
+#     factual_claims = len(factual_sentences)
+
+#     claims_with_sources = sum(
+#         1 for s in factual_sentences if SOURCE_ATTRIBUTION_PATTERNS.search(s)
+#     )
+#     claims_with_sources_ratio = _pct(claims_with_sources, max(1, factual_claims)) if factual_claims else None
+
+#     original_info_hits = len(FIRST_HAND_PATTERNS.findall(text or ""))
+#     original_information = _clamp(min(100, original_info_hits * 25))
+#     first_hand_experience = original_information
+
+#     return {
+#         "factual_claims": factual_claims,
+#         "claims_with_sources": claims_with_sources,
+#         "claims_with_sources_ratio": claims_with_sources_ratio,
+#         "original_information": original_information,
+#         "first_hand_experience": first_hand_experience,
+#     }
+
+
+# # ==============================
+# # CATEGORY 5: TRUST / AUTHORITY
+# # ==============================
+# def _analyze_trust(seo_data, soup, text, links, page_type):
+#     author_relevant = page_type in TYPES_WHERE_AUTHOR_RELEVANT
+
+#     author_bio_hit = bool(re.search(r'\b(PhD|M\.?D\.?|certified|expert|years of experience|founder|CEO|specialist)\b', text or "", re.I))
+#     if author_relevant:
+#         author_expertise = 70 if author_bio_hit else 0
+#         author_expertise_status = "measured"
+#     else:
+#         author_expertise = None
+#         author_expertise_status = "not_applicable"
+#     author_credentials = author_bio_hit
+
+#     hrefs = [l.get("href", "") for l in links]
+#     about_page_present = any(re.search(r'/about', h, re.I) for h in hrefs)
+#     contact_page_present = any(re.search(r'/contact', h, re.I) for h in hrefs)
+#     email_present = bool(re.search(r'[\w.+-]+@[\w-]+\.[\w.-]+', text or ""))
+#     phone_present = bool(re.search(r'(\+?\d[\d\s().-]{7,}\d)', text or ""))
+#     contact_information_present = contact_page_present or email_present or phone_present
+
+#     privacy_present = any(re.search(r'privacy', h, re.I) for h in hrefs)
+#     terms_present = any(re.search(r'terms', h, re.I) for h in hrefs)
+#     https_present = str(seo_data.get("url", "")).startswith("https://")
+
+#     trust_signal_flags = [about_page_present, contact_information_present, privacy_present, terms_present, https_present]
+#     trust_signals = sum(trust_signal_flags)
+
+#     organization_transparency = _clamp(_pct(trust_signals, len(trust_signal_flags)))
+
+#     return {
+#         "author_expertise": author_expertise,
+#         "author_expertise_status": author_expertise_status,
+#         "author_credentials": author_credentials if author_relevant else None,
+#         "organization_transparency": organization_transparency,
+#         "about_page_present": about_page_present,
+#         "contact_information_present": contact_information_present,
+#         "trust_signals": trust_signals,
+#     }
+
+
+# # ==============================
+# # CATEGORY 6: STRUCTURED DATA (supporting signal, not dominant)
+# # ==============================
+# def _extract_schema_types(seo_data, soup):
+#     schemas = seo_data.get("schemas", [])
+#     schema_types = []
+#     schema_valid = True
+
+#     for schema in schemas:
+#         try:
+#             if isinstance(schema, dict):
+#                 if "@type" in schema:
+#                     t = schema["@type"]
+#                     schema_types.extend(t if isinstance(t, list) else [t])
+#                 if "@graph" in schema and isinstance(schema["@graph"], list):
+#                     for item in schema["@graph"]:
+#                         if isinstance(item, dict) and "@type" in item:
+#                             schema_types.append(item["@type"])
+#             elif isinstance(schema, list):
+#                 for item in schema:
+#                     if isinstance(item, dict) and "@type" in item:
+#                         schema_types.append(item["@type"])
+#         except Exception:
+#             schema_valid = False
+
+#     schema_types = list(dict.fromkeys(str(t) for t in schema_types))
+#     return schema_types, schema_valid
+
+
+# def _analyze_structured_data(seo_data, schema_types, schema_valid, page_type):
+#     lower_types = [t.lower() for t in schema_types]
+#     schema_present = len(schema_types) > 0
+#     organization_schema = any("organization" in t for t in lower_types)
+#     article_schema = any("article" in t for t in lower_types)
+#     product_schema = any("product" in t for t in lower_types)
+#     faq_schema = any("faq" in t for t in lower_types)
+#     breadcrumb_schema = any("breadcrumb" in t for t in lower_types)
+
+#     # completeness: how many of the schema types *relevant to this page type*
+#     # are present, rather than expecting every type on every page.
+#     relevant_types = {"organization"}
+#     if page_type in TYPES_WHERE_ARTICLE_SCHEMA_RELEVANT:
+#         relevant_types.add("article")
+#     if page_type in TYPES_WHERE_PRODUCT_SCHEMA_RELEVANT:
+#         relevant_types.add("product")
+#     if page_type in TYPES_WHERE_FAQ_RELEVANT:
+#         relevant_types.add("faq")
+#     present_map = {
+#         "organization": organization_schema, "article": article_schema,
+#         "product": product_schema, "faq": faq_schema,
+#     }
+#     relevant_present = sum(1 for t in relevant_types if present_map.get(t))
+#     schema_completeness = _clamp(round(_pct(relevant_present, max(1, len(relevant_types)))))
+
+#     org_schema_name = None
+#     for schema in seo_data.get("schemas", []) or []:
+#         if isinstance(schema, dict) and str(schema.get("@type", "")).lower() == "organization":
+#             org_schema_name = schema.get("name")
+#             break
+#     domain = urlparse(seo_data.get("url", "")).netloc.replace("www.", "").split(".")[0]
+#     schema_entity_alignment = bool(
+#         org_schema_name and domain and domain.lower() in str(org_schema_name).lower()
+#     )
+
+#     return {
+#         "schema_present": schema_present,
+#         "schema_types": ", ".join(schema_types) if schema_types else "not_detected",
+#         "schema_valid": schema_valid,
+#         "schema_completeness": schema_completeness,
+#         "schema_entity_alignment": schema_entity_alignment,
+#         "organization_schema": organization_schema,
+#         "article_schema": article_schema,
+#         "product_schema": product_schema,
+#         "faq_schema": faq_schema,
+#         "breadcrumb_schema": breadcrumb_schema,
+#     }
+
+
+# # ==============================
+# # CATEGORY 7: RETRIEVAL / CRAWLABILITY
+# # ==============================
+# def _fetch_raw_html_sync(url, timeout=6):
+#     """Cheap plain-HTTP GET (no browser) used only to compare against the
+#     Playwright-rendered HTML, so we can tell whether critical content is
+#     server-rendered or injected by JavaScript. This is a single lightweight
+#     request per page, not a second full crawl."""
+#     try:
+#         req = Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; AIVisibilityBot/1.0)"})
+#         with urlopen(req, timeout=timeout) as resp:
+#             raw = resp.read(2_000_000)
+#         return raw.decode("utf-8", errors="ignore")
+#     except Exception:
+#         return None
+
+
+# async def _analyze_retrieval(seo_data, word_count, text, html):
+#     robots = (seo_data.get("robots") or "").lower()
+#     indexable = "noindex" not in robots
+#     robots_allowed = indexable  # proxy: only meta robots is checked, robots.txt is not fetched
+
+#     canonical = seo_data.get("canonical") or ""
+#     domain = urlparse(seo_data.get("url", "")).netloc
+#     canonical_valid = bool(canonical) and (domain == "" or domain in canonical)
+
+#     content_accessible = word_count > 0
+#     renderable_content = word_count >= 50
+
+#     url = seo_data.get("url", "")
+#     raw_html = None
+#     if url:
+#         raw_html = await asyncio.get_event_loop().run_in_executor(None, _fetch_raw_html_sync, url)
+
+#     content_rendering = {
+#         "raw_word_count": None,
+#         "rendered_word_count": word_count,
+#         "rendering_dependency_ratio": None,
+#         "critical_content_server_rendered": None,
+#         "retrieval_status": "unknown",
+#     }
+
+#     if raw_html is not None:
+#         raw_soup = BeautifulSoup(raw_html, "html.parser")
+#         raw_text = raw_soup.get_text(separator=" ", strip=True)
+#         raw_word_count = len(_words(raw_text))
+#         content_rendering["raw_word_count"] = raw_word_count
+#         ratio = _pct(raw_word_count, max(1, word_count))
+#         content_rendering["rendering_dependency_ratio"] = ratio
+#         content_rendering["critical_content_server_rendered"] = raw_word_count >= 50 and ratio >= 60
+#         if not content_accessible:
+#             content_rendering["retrieval_status"] = "poor"
+#         elif content_rendering["critical_content_server_rendered"]:
+#             content_rendering["retrieval_status"] = "good"
+#         elif raw_word_count >= 50:
+#             content_rendering["retrieval_status"] = "limited"
+#         else:
+#             content_rendering["retrieval_status"] = "js_dependent"
+#     else:
+#         # Couldn't do the plain-HTTP comparison (blocked, timeout, etc).
+#         # We do NOT penalize the page for this - it's a measurement gap.
+#         content_rendering["retrieval_status"] = "unknown"
+
+#     return {
+#         "indexable": indexable,
+#         "robots_allowed": robots_allowed,
+#         "canonical_valid": canonical_valid,
+#         "content_accessible": content_accessible,
+#         "renderable_content": renderable_content,
+#         "content_rendering": content_rendering,
+#     }
+
+
+# # ==============================
+# # CATEGORY 8: CONTENT STRUCTURE
+# # ==============================
+# def _analyze_content_structure(soup):
+#     heading_tags = soup.find_all(re.compile("^h[1-6]$"))
+#     levels = []
+#     for h in heading_tags:
+#         try:
+#             levels.append(int(h.name[1]))
+#         except Exception:
+#             continue
+
+#     heading_structure_score = 0
+#     if 1 in levels:
+#         heading_structure_score += 40
+#     if 2 in levels:
+#         heading_structure_score += 30
+#     if levels == sorted(levels):
+#         heading_structure_score += 30
+#     heading_structure_score = _clamp(heading_structure_score)
+
+#     paragraphs = [p.get_text(" ", strip=True) for p in soup.find_all("p")]
+#     paragraphs = [p for p in paragraphs if p]
+#     if paragraphs:
+#         avg_len = sum(len(_words(p)) for p in paragraphs) / len(paragraphs)
+#         paragraph_clarity = 100 if 15 <= avg_len <= 40 else _clamp(100 - abs(avg_len - 27) * 3)
+#     else:
+#         paragraph_clarity = 0
+
+#     list_usage = len(soup.find_all(["ul", "ol"])) > 0
+#     table_usage = len(soup.find_all("table")) > 0
+#     definition_sections = bool(soup.find("dl")) or bool(soup.find("dfn"))
+#     summary_present = bool(soup.find(
+#         lambda t: t.name in ("h1", "h2", "h3") and re.search(r"summary|tl;?dr|key takeaways", t.get_text(" ", strip=True), re.I)
+#     ))
+
+#     return {
+#         "heading_structure_score": heading_structure_score,
+#         "paragraph_clarity": round(paragraph_clarity),
+#         "list_usage": list_usage,
+#         "table_usage": table_usage,
+#         "definition_sections": definition_sections,
+#         "summary_present": summary_present,
+#     }
+
+
+# # ==============================
+# # CATEGORY 9: CITATION POTENTIAL
+# # (based on original data/quotes/attribution actually present in the
+# # content, not on raw external-link counting)
+# # ==============================
+# def _analyze_citation_potential(soup, text):
+#     numbers = re.findall(r'\b\d+(?:[.,]\d+)?%?\b', text or "")
+#     unique_data_points = len(set(numbers))
+#     statistics_present = unique_data_points > 0
+
+#     original_research = bool(FIRST_HAND_PATTERNS.search(text or ""))
+
+#     blockquotes = soup.find_all("blockquote")
+#     quoted_sentences = re.findall(r'"[^"]{20,200}"', text or "")
+#     quotable_statements = len(blockquotes) + len(quoted_sentences)
+
+#     source_attribution = len(SOURCE_ATTRIBUTION_PATTERNS.findall(text or ""))
+
+#     score = 0
+#     score += min(35, unique_data_points * 3)
+#     score += 25 if original_research else 0
+#     score += min(20, quotable_statements * 5)
+#     score += min(20, source_attribution * 10)
+#     citation_potential = _clamp(score)
+
+#     return {
+#         "unique_data_points": unique_data_points,
+#         "statistics_present": statistics_present,
+#         "original_research": original_research,
+#         "quotable_statements": quotable_statements,
+#         "source_attribution": source_attribution,
+#         "citation_potential": citation_potential,
+#     }
+
+
+# # ==============================
+# # CATEGORY 10: FRESHNESS (fresh / stale / very_stale / unknown - never
+# # "guessed outdated")
+# # ==============================
+# def _analyze_freshness(seo_data, soup, text, page_type):
+#     metas = seo_data.get("metas", {}) or {}
+#     last_updated = (
+#         metas.get("article:modified_time")
+#         or metas.get("article:published_time")
+#         or metas.get("date")
+#     )
+
+#     if not last_updated:
+#         time_tag = soup.find("time", attrs={"datetime": True})
+#         if time_tag:
+#             last_updated = time_tag.get("datetime")
+
+#     if not last_updated:
+#         for pattern in DATE_PATTERNS:
+#             match = pattern.search(text or "")
+#             if match:
+#                 last_updated = match.group(0)
+#                 break
+
+#     date_visible = bool(last_updated)
+#     content_age_days = None
+
+#     if last_updated:
+#         try:
+#             from datetime import datetime, timezone
+#             parsed = None
+#             for fmt in ("%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
+#                 try:
+#                     parsed = datetime.strptime(last_updated[:19].replace("Z", ""), fmt)
+#                     break
+#                 except Exception:
+#                     continue
+#             if parsed:
+#                 parsed = parsed.replace(tzinfo=timezone.utc)
+#                 content_age_days = (datetime.now(timezone.utc) - parsed).days
+#         except Exception:
+#             content_age_days = None
+
+#     if not date_visible:
+#         freshness_status = "unknown"
+#         freshness_reason = "No reliable publication/update date detected on the page."
+#     elif isinstance(content_age_days, int):
+#         # FIX: previously the 365< age <=730 and >730 branches both produced
+#         # "stale", making the 730-day boundary dead code. Now a genuine
+#         # "very_stale" tier exists for content older than 2 years.
+#         if content_age_days <= 365:
+#             freshness_status = "fresh"
+#         elif content_age_days <= 730:
+#             freshness_status = "stale"
+#         else:
+#             freshness_status = "very_stale"
+#         freshness_reason = f"Date detected: content is {content_age_days} days old."
+#     else:
+#         freshness_status = "unknown"
+#         freshness_reason = "A date-like string was found but could not be reliably parsed."
+
+#     return {
+#         "last_updated": last_updated or "not_detected",
+#         "content_age_days": content_age_days,
+#         "update_frequency": "not_detected",  # requires historical crawl data, unavailable here
+#         "date_visible": date_visible,
+#         "freshness_status": freshness_status,
+#         "freshness_reason": freshness_reason,
+#         "freshness_relevant": page_type in TYPES_WHERE_FRESHNESS_RELEVANT,
+#     }
+
+
+# # ==============================
+# # CATEGORY 11: BRAND / ENTITY CONSISTENCY
+# # ==============================
+# def _analyze_brand_consistency(seo_data, soup, text, schema_types):
+#     domain = urlparse(seo_data.get("url", "")).netloc.replace("www.", "")
+#     brand_guess = domain.split(".")[0] if domain else ""
+
+#     title = (seo_data.get("title") or "")
+#     footer = soup.find("footer")
+#     footer_text = footer.get_text(" ", strip=True) if footer else ""
+
+#     brand_in_title = bool(brand_guess) and brand_guess.lower() in title.lower()
+#     brand_in_footer = bool(brand_guess) and brand_guess.lower() in footer_text.lower()
+#     brand_in_body = bool(brand_guess) and brand_guess.lower() in (text or "").lower()
+#     brand_name_consistency = _clamp(sum([brand_in_title, brand_in_footer, brand_in_body]) * 33)
+
+#     org_schema_name = None
+#     for schema in seo_data.get("schemas", []) or []:
+#         if isinstance(schema, dict) and str(schema.get("@type", "")).lower() == "organization":
+#             org_schema_name = schema.get("name")
+#             break
+
+#     company_information_consistency = _clamp(
+#         70 if (org_schema_name and brand_guess and brand_guess.lower() in str(org_schema_name).lower()) else (30 if brand_in_footer else 0)
+#     )
+
+#     author_meta = (seo_data.get("metas", {}) or {}).get("author", "")
+#     byline = soup.find(class_=re.compile("author|byline", re.I))
+#     byline_text = byline.get_text(" ", strip=True) if byline else ""
+#     author_information_consistency = _clamp(
+#         70 if (author_meta and byline_text and author_meta.lower() in byline_text.lower())
+#         else (40 if (author_meta or byline_text) else 0)
+#     )
+
+#     return {
+#         "brand_name_consistency": brand_name_consistency,
+#         "company_information_consistency": company_information_consistency,
+#         "author_information_consistency": author_information_consistency,
+#     }
+
+
+# # ==============================
+# # CATEGORY SCORE ROLLUPS (unknown values excluded, weight redistributed)
+# # ==============================
+# def _avg_known(*values):
+#     """Average only the non-None values. Returns None if all are unknown -
+#     the caller decides how to treat that (usually: exclude from rollup)."""
+#     known = [v for v in values if v is not None]
+#     if not known:
+#         return None
+#     return round(sum(known) / len(known))
+
+
+# def _rollup_scores(m, page_type):
+#     weights = get_weights_for_page_type(page_type)
+
+#     semantic_parts = [m["topic_clarity"], m["topic_coherence"], m["content_completeness"], m["content_depth"]]
+#     if m.get("semantic_relevance") is not None:
+#         semantic_parts.append(m["semantic_relevance"])
+#     if m.get("search_intent_match") is not None:
+#         semantic_parts.append(m["search_intent_match"])
+#     semantic = _avg_known(*semantic_parts)
+
+#     content_answerability = _avg_known(
+#         m["content_completeness"],
+#         m["answer_coverage"],
+#         100 if m["direct_answer_presence"] else 0,
+#         100 if m["definition_presence"] else 0,
+#         m["faq_coverage"] if page_type in TYPES_WHERE_FAQ_RELEVANT or m["question_count"] > 0 else None,
+#         _clamp(min(100, m["factual_claims"] * 10)),
+#     )
+
+#     entity_parts = [m["entity_consistency"]]
+#     if m.get("entity_clarity") is not None:
+#         entity_parts.append(m["entity_clarity"])
+#     entity_parts.append(100 if m["organization_entity_present"] else 0)
+#     if page_type in TYPES_WHERE_AUTHOR_RELEVANT:
+#         entity_parts.append(100 if m["author_entity_present"] else 0)
+#     entity = _avg_known(*entity_parts)
+
+#     trust_parts = [
+#         m["organization_transparency"],
+#         100 if m["about_page_present"] else 0,
+#         100 if m["contact_information_present"] else 0,
+#     ]
+#     if m.get("author_expertise") is not None:
+#         trust_parts.append(m["author_expertise"])
+#     trust = _avg_known(*trust_parts)
+
+#     citation = m["citation_potential"]
+
+#     retrieval_parts = [
+#         100 if m["indexable"] else 0,
+#         100 if m["canonical_valid"] else 0,
+#         100 if m["content_accessible"] else 0,
+#     ]
+#     rendering_status = m.get("content_rendering", {}).get("retrieval_status")
+#     if rendering_status == "good":
+#         retrieval_parts.append(100)
+#     elif rendering_status == "limited":
+#         retrieval_parts.append(60)
+#     elif rendering_status == "js_dependent":
+#         retrieval_parts.append(30)
+#     # "unknown" contributes nothing - not penalized, not rewarded.
+#     retrieval = _avg_known(*retrieval_parts)
+
+#     schema_hits = sum([
+#         m["organization_schema"], m["article_schema"], m["product_schema"],
+#         m["faq_schema"], m["breadcrumb_schema"],
+#     ])
+#     structured_data = _clamp(
+#         m["schema_completeness"] * 0.6 + (20 if m["schema_valid"] and m["schema_present"] else 0) + schema_hits * 4
+#     )
+
+#     if m["freshness_status"] == "fresh":
+#         if isinstance(m["content_age_days"], int):
+#             freshness = 100 if m["content_age_days"] <= 90 else 75
+#         else:
+#             freshness = 75
+#     elif m["freshness_status"] == "stale":
+#         freshness = 30
+#     elif m["freshness_status"] == "very_stale":
+#         freshness = 10
+#     else:
+#         freshness = None  # unknown - excluded from weighted rollup entirely
+
+#     components = {
+#         "semantic": semantic,
+#         "content_answerability": content_answerability,
+#         "entity": entity,
+#         "trust": trust,
+#         "citation": citation,
+#         "retrieval": retrieval,
+#         "structured_data": structured_data,
+#         "freshness": freshness,
+#     }
+
+#     # Weighted average over KNOWN components only; unknown components'
+#     # weight is redistributed proportionally rather than counted as 0.
+#     known_weight = sum(weights[k] for k, v in components.items() if v is not None)
+#     if known_weight <= 0:
+#         ai_readiness_score = 0
+#     else:
+#         ai_readiness_score = round(
+#             sum(components[k] * weights[k] for k in components if components[k] is not None) / known_weight
+#         )
+#     ai_readiness_score = _clamp(ai_readiness_score)
+
+#     return {
+#         "semantic_score": semantic,
+#         "content_answerability_score": content_answerability,
+#         "entity_score": entity,
+#         "trust_score": trust,
+#         "citation_potential_score": citation,
+#         "retrieval_score": retrieval,
+#         "structured_data_score": structured_data,
+#         "freshness_score": freshness,
+#         "ai_readiness_score": ai_readiness_score,
+#         # legacy alias kept for the existing frontend/API consumers
+#         "ai_visibility_score": ai_readiness_score,
+#         "weights_used": weights,
+#     }
+
+
+# # ==============================
+# # OPTIONAL LLM ENHANCEMENT (semantic-only, reuses existing OpenAI setup)
+# # ==============================
+# async def _llm_semantic_enhance(seo_data, text, deterministic, target_query=None):
+#     # FIX: previously used the removed openai<1.0 `openai.ChatCompletion.create`
+#     # API, which raises AttributeError on any openai-python>=1.0 install and
+#     # silently fell back to deterministic-only scoring every time. Now uses
+#     # the current client-based API (openai.OpenAI().chat.completions.create).
+#     client = _get_openai_client()
+#     if not OPENAI_AVAILABLE or client is None:
+#         return None
+
+#     excerpt = (text or "")[:2000]
+#     query_line = f"TARGET QUERY: {target_query}\n" if target_query else ""
+#     prompt = f"""You are assessing AI-search readiness of a web page (not traditional SEO).
+# Given the page title, meta description and a content excerpt, score each item 0-100 based on how easily
+# an AI system could understand and summarize this page. Return ONLY valid JSON with these exact keys:
+# topic_clarity, content_completeness, entity_clarity, citation_potential, original_information{"," if target_query else ""}
+# {"semantic_relevance, search_intent_match" if target_query else ""}
+
+# {query_line}TITLE: {seo_data.get('title', '')}
+# META DESCRIPTION: {seo_data.get('description', '')}
+# CONTENT EXCERPT: {excerpt}
+# """
+#     try:
+#         response = await asyncio.get_event_loop().run_in_executor(
+#             None,
+#             lambda: client.chat.completions.create(
+#                 model="gpt-4o-mini",
+#                 messages=[
+#                     {"role": "system", "content": "You output only strict JSON, no prose, no markdown fences."},
+#                     {"role": "user", "content": prompt},
+#                 ],
+#                 max_tokens=300,
+#                 temperature=0.3,
+#             ),
+#         )
+#         raw = response.choices[0].message.content.strip()
+#         raw = re.sub(r"^```(json)?|```$", "", raw.strip(), flags=re.I).strip()
+#         data = json.loads(raw)
+#         keys = ["topic_clarity", "content_completeness", "entity_clarity", "citation_potential", "original_information"]
+#         if target_query:
+#             keys += ["semantic_relevance", "search_intent_match"]
+#         cleaned = {}
+#         for k in keys:
+#             v = data.get(k)
+#             if isinstance(v, (int, float)):
+#                 cleaned[k] = _clamp(round(v))
+#         return cleaned or None
+#     except Exception as e:
+#         print(f"   AI visibility LLM enhancement failed: {str(e)[:120]}")
+#         return None
+
+
+# def _blend(deterministic_value, llm_value):
+#     if llm_value is None:
+#         return deterministic_value
+#     if deterministic_value is None:
+#         return llm_value
+#     return round((deterministic_value + llm_value) / 2)
+
+
+# # ==============================
+# # PER-PAGE ORCHESTRATION
+# # ==============================
+# async def analyze_page_ai_visibility(seo_data, domain, use_ai=False, target_query=None):
+#     """Compute the full ai_visibility metric set for one already-fetched page."""
+#     empty_scores = {
+#         "semantic_score": None, "content_answerability_score": None, "entity_score": None,
+#         "trust_score": None, "citation_potential_score": None, "retrieval_score": None,
+#         "structured_data_score": None, "freshness_score": None,
+#         "ai_readiness_score": 0, "ai_visibility_score": 0, "weights_used": {},
+#     }
+#     try:
+#         html = seo_data.get("html", "")
+#         soup = BeautifulSoup(html, "html.parser")
+#         text = soup.get_text(separator=" ", strip=True)
+#         word_count = len(_words(text))
+#         links = seo_data.get("links", [])
+
+#         schema_types, schema_valid = _extract_schema_types(seo_data, soup)
+#         page_type_info = classify_page_type(seo_data, soup, text, word_count, schema_types, links)
+#         page_type = page_type_info["type"]
+
+#         m = {}
+#         m.update(_analyze_semantic(seo_data, soup, text, word_count, target_query))
+#         m.update(_analyze_entities(seo_data, soup, text, word_count, schema_types, page_type))
+#         m.update(_analyze_answerability(soup, text))
+#         m.update(_analyze_information_quality(soup, text))
+#         m.update(_analyze_trust(seo_data, soup, text, links, page_type))
+#         m.update(_analyze_structured_data(seo_data, schema_types, schema_valid, page_type))
+#         m.update(await _analyze_retrieval(seo_data, word_count, text, html))
+#         m.update(_analyze_content_structure(soup))
+#         m.update(_analyze_citation_potential(soup, text))
+#         m.update(_analyze_freshness(seo_data, soup, text, page_type))
+#         m.update(_analyze_brand_consistency(seo_data, soup, text, schema_types))
+
+#         if use_ai:
+#             llm_result = await _llm_semantic_enhance(seo_data, text, m, target_query)
+#             if llm_result:
+#                 for k, v in llm_result.items():
+#                     if k in m:
+#                         m[k] = _blend(m[k], v)
+
+#         scores = _rollup_scores(m, page_type)
+
+#         # Build unknown/not-applicable metric lists for transparency.
+#         unknown_metrics = []
+#         if m.get("semantic_relevance_status") == "requires_target_query":
+#             unknown_metrics.append({"metric": "semantic_relevance", "reason": "No target keyword/query was supplied for this analysis."})
+#         if m.get("freshness_status") == "unknown":
+#             unknown_metrics.append({"metric": "freshness", "reason": m.get("freshness_reason", "No date detected.")})
+#         if m.get("author_expertise_status") == "not_applicable":
+#             unknown_metrics.append({"metric": "author_expertise", "reason": f"Not applicable for page type '{page_type}'."})
+#         if m.get("content_rendering", {}).get("retrieval_status") == "unknown":
+#             unknown_metrics.append({"metric": "retrieval_rendering", "reason": "Could not fetch raw HTML to compare against rendered content."})
+
+#         page_ai_visibility = {
+#             "page_type": page_type,
+#             "page_type_confidence": page_type_info["confidence"],
+#             "topic_clarity": m["topic_clarity"],
+#             "semantic_relevance": m["semantic_relevance"],
+#             "content_completeness": m["content_completeness"],
+#             "entity_clarity": m["entity_clarity"],
+#             "answer_coverage": m["answer_coverage"],
+#             "factual_information": m["factual_claims"],
+#             "original_information": m["original_information"],
+#             "author_expertise": m["author_expertise"],
+#             "schema_quality": scores["structured_data_score"],
+#             "crawlability": scores["retrieval_score"],
+#             "content_structure": m["heading_structure_score"],
+#             "citation_potential": m["citation_potential"],
+#             "freshness_status": m["freshness_status"],
+#             "freshness": scores["freshness_score"],
+#             "ai_readiness_score": scores["ai_readiness_score"],
+#             "ai_visibility_score": scores["ai_visibility_score"],
+#         }
+
+#         return {
+#             "url": seo_data.get("url", ""),
+#             "title": seo_data.get("title", ""),
+#             "page_type": page_type,
+#             "page_type_confidence": page_type_info["confidence"],
+#             "raw_metrics": m,
+#             "scores": scores,
+#             "unknown_metrics": unknown_metrics,
+#             "ai_visibility": page_ai_visibility,
+#         }
+#     except Exception as e:
+#         print(f"AI visibility analysis error for {seo_data.get('url', 'unknown')}: {e}")
+#         return {
+#             "url": seo_data.get("url", ""),
+#             "title": seo_data.get("title", ""),
+#             "page_type": "unknown",
+#             "page_type_confidence": 0,
+#             "raw_metrics": {},
+#             "scores": empty_scores,
+#             "unknown_metrics": [],
+#             "ai_visibility": {
+#                 "page_type": "unknown", "page_type_confidence": 0,
+#                 "topic_clarity": 0, "semantic_relevance": None, "content_completeness": 0,
+#                 "entity_clarity": None, "answer_coverage": None, "factual_information": 0,
+#                 "original_information": 0, "author_expertise": None, "schema_quality": 0,
+#                 "crawlability": 0, "content_structure": 0, "citation_potential": 0,
+#                 "freshness_status": "unknown", "freshness": None,
+#                 "ai_readiness_score": 0, "ai_visibility_score": 0,
+#             },
+#             "error": str(e),
+#         }
+
+
+# # ==============================
+# # ISSUE / STRENGTH GENERATION (evidence-based, page-type-aware)
+# # ==============================
+# def _build_issues_and_strengths(page_results):
+#     issues = []
+#     strengths = []
+#     unknowns = []
+
+#     for p in page_results:
+#         s = p["scores"]
+#         m = p["raw_metrics"]
+#         url = p["url"]
+#         page_type = p.get("page_type", "unknown")
+#         if not m:
+#             continue
+
+#         # --- Answerability: only relevant where questions/FAQ genuinely matter ---
+#         if page_type in TYPES_WHERE_FAQ_RELEVANT or m.get("question_count", 0) > 0:
+#             ca_score = s.get("content_answerability_score")
+#             if ca_score is not None and ca_score < 50:
+#                 issues.append({
+#                     "title": "Missing clear answers to common questions",
+#                     "severity": "high" if ca_score < 25 else "medium",
+#                     "page": url, "metric": "content_answerability_score", "current_value": ca_score,
+#                     "explanation": f"Content & answerability score is {ca_score}/100 for this {page_type} page - questions are not clearly answered, or no question-style headings/FAQ exist.",
+#                     "recommended_fix": "Add direct, concise answers (40-300 chars) immediately after question-style headings, and consider an FAQ section.",
+#                 })
+#             elif ca_score is not None and ca_score >= 70:
+#                 strengths.append({"page": url, "title": "Strong answerability", "detail": f"Content & answerability score {ca_score}/100."})
+
+#         # --- Entity ---
+#         if s.get("entity_score") is not None:
+#             if s["entity_score"] < 40:
+#                 issues.append({
+#                     "title": "Weak entity information",
+#                     "severity": "medium", "page": url, "metric": "entity_score", "current_value": s["entity_score"],
+#                     "explanation": f"Entity score is {s['entity_score']}/100 - the page doesn't clearly establish named entities (organizations, people, products) relevant to a '{page_type}' page.",
+#                     "recommended_fix": "Mention your organization/brand and product names explicitly and consistently, and add Organization/Person schema.",
+#                 })
+#             elif s["entity_score"] >= 70:
+#                 strengths.append({"page": url, "title": "Strong entity clarity", "detail": f"Entity score {s['entity_score']}/100."})
+
+#         # --- Trust / author (only where relevant to page type) ---
+#         if page_type in TYPES_WHERE_AUTHOR_RELEVANT:
+#             if not m.get("author_entity_present") or (m.get("author_expertise") in (0, None)):
+#                 issues.append({
+#                     "title": "No author expertise information detected",
+#                     "severity": "medium", "page": url, "metric": "author_expertise", "current_value": m.get("author_expertise"),
+#                     "explanation": f"No author byline or credentials were found on this {page_type} page, where authorship signals matter for trust.",
+#                     "recommended_fix": "Add a visible author byline with credentials, or an author bio linking to their expertise.",
+#                 })
+
+#         # --- Citation potential ---
+#         if s.get("citation_potential_score") is not None and s["citation_potential_score"] < 40:
+#             issues.append({
+#                 "title": "Low citation potential",
+#                 "severity": "low", "page": url, "metric": "citation_potential_score", "current_value": s["citation_potential_score"],
+#                 "explanation": f"Citation potential score is {s['citation_potential_score']}/100 - the page has few unique statistics, quotes, or attributed sources an AI system could cite.",
+#                 "recommended_fix": "Add original statistics, data points, or quotable expert statements with clear sourcing.",
+#             })
+#         elif s.get("citation_potential_score", 0) >= 70:
+#             strengths.append({"page": url, "title": "Strong citation potential", "detail": f"Citation potential score {s['citation_potential_score']}/100."})
+
+#         # --- Retrieval / JS dependency: only flagged when there's actual evidence content is JS-gated ---
+#         rendering = m.get("content_rendering", {})
+#         if rendering.get("retrieval_status") == "js_dependent":
+#             issues.append({
+#                 "title": "Critical content is not present in server-rendered HTML",
+#                 "severity": "medium", "page": url, "metric": "retrieval_rendering",
+#                 "current_value": rendering.get("rendering_dependency_ratio"),
+#                 "explanation": f"Raw (non-JS) fetch returned only {rendering.get('raw_word_count', 0)} words vs {rendering.get('rendered_word_count', 0)} rendered - most content is injected by JavaScript, which some AI crawlers do not execute.",
+#                 "recommended_fix": "Ensure key content is present in server-rendered HTML (SSR) or a no-JS fallback.",
+#             })
+#         elif rendering.get("retrieval_status") == "good":
+#             strengths.append({"page": url, "title": "Content is server-rendered", "detail": "Critical text content is present without executing JavaScript."})
+
+#         # --- Content structure ---
+#         if m.get("heading_structure_score", 0) < 50:
+#             issues.append({
+#                 "title": "Poor content structure",
+#                 "severity": "low", "page": url, "metric": "heading_structure_score", "current_value": m.get("heading_structure_score", 0),
+#                 "explanation": f"Heading structure score is {m.get('heading_structure_score', 0)}/100.",
+#                 "recommended_fix": "Use a single H1 followed by a logical H2/H3 hierarchy so AI systems can segment the content.",
+#             })
+
+#         # --- Structured data: only an issue when a relevant schema type is actually missing ---
+#         relevant_missing = []
+#         if page_type in TYPES_WHERE_ARTICLE_SCHEMA_RELEVANT and not m.get("article_schema"):
+#             relevant_missing.append("Article")
+#         if page_type in TYPES_WHERE_PRODUCT_SCHEMA_RELEVANT and not m.get("product_schema"):
+#             relevant_missing.append("Product")
+#         if page_type in TYPES_WHERE_FAQ_RELEVANT and m.get("question_count", 0) >= 2 and not m.get("faq_schema"):
+#             relevant_missing.append("FAQPage")
+#         if relevant_missing:
+#             issues.append({
+#                 "title": "Missing structured data",
+#                 "severity": "medium", "page": url, "metric": "schema_present", "current_value": m.get("schema_types"),
+#                 "explanation": f"No {'/'.join(relevant_missing)} schema was detected, though it is relevant for a '{page_type}' page.",
+#                 "recommended_fix": f"Add {'/'.join(relevant_missing)} JSON-LD structured data.",
+#             })
+
+#         # --- Freshness: unknown is reported separately, never as "outdated" ---
+#         if m.get("freshness_status") == "unknown" and page_type in TYPES_WHERE_FRESHNESS_RELEVANT:
+#             unknowns.append({"page": url, "metric": "freshness", "reason": "No publish/update date could be detected on this page, where freshness is typically relevant."})
+#         elif m.get("freshness_status") in ("stale", "very_stale"):
+#             issues.append({
+#                 "title": "Content appears stale",
+#                 "severity": "low" if m.get("freshness_status") == "stale" else "medium",
+#                 "page": url, "metric": "freshness_status", "current_value": m.get("content_age_days"),
+#                 "explanation": f"A reliable date was detected and the content is {m.get('content_age_days')} days old.",
+#                 "recommended_fix": "Review and update the content, then refresh the visible date / article:modified_time meta tag.",
+#             })
+
+#         for u in p.get("unknown_metrics", []):
+#             unknowns.append({"page": url, **u})
+
+#     severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+#     issues.sort(key=lambda x: severity_order.get(x["severity"], 4))
+#     return issues[:30], strengths[:20], unknowns[:30]
+
+
+# # ==============================
+# # TOP-LEVEL ENTRYPOINT
+# # ==============================
+# async def run_ai_visibility_analysis(base_url, max_pages=5, max_concurrent=1, use_ai=False, target_query=None):
+#     """
+#     Fetches pages (once) and computes AI-readiness metrics for each.
+
+#     Returns a dict with an `ai_readiness_score` (transparent 0-100 proxy
+#     score for how well pages are prepared to be understood/cited by AI
+#     systems) plus per-page results, page-type classification, issues,
+#     strengths, and unknown metrics. `actual_ai_visibility` is always
+#     "not_measured" - this crawler has no access to real AI-query/citation
+#     data, so it never fabricates one.
+#     """
+#     if not base_url:
+#         raise ValueError("base_url is required")
+
+#     domain = urlparse(base_url).netloc
+
+#     urls = await discover_urls_parallel(base_url, max_pages)
+#     if not urls:
+#         urls = [base_url]
+
+#     playwright_data = await fetch_all_pages_parallel(urls, max_concurrent)
+#     if not playwright_data:
+#         return {
+#             "status": "error",
+#             "message": "Failed to fetch any pages. Site may be blocking bots or require authentication.",
+#         }
+
+#     page_results = []
+#     for seo_data in playwright_data:
+#         result = await analyze_page_ai_visibility(seo_data, domain, use_ai=use_ai, target_query=target_query)
+#         page_results.append(result)
+
+#     valid_scores = [p["scores"]["ai_readiness_score"] for p in page_results if p.get("raw_metrics")]
+#     overall_score = round(sum(valid_scores) / len(valid_scores)) if valid_scores else 0
+
+#     def _avg(key):
+#         vals = [p["scores"][key] for p in page_results if p.get("raw_metrics") and p["scores"].get(key) is not None]
+#         return round(sum(vals) / len(vals)) if vals else None
+
+#     category_averages = {
+#         "semantic_score": _avg("semantic_score"),
+#         "content_answerability_score": _avg("content_answerability_score"),
+#         "entity_score": _avg("entity_score"),
+#         "trust_score": _avg("trust_score"),
+#         "citation_potential_score": _avg("citation_potential_score"),
+#         "retrieval_score": _avg("retrieval_score"),
+#         "structured_data_score": _avg("structured_data_score"),
+#         "freshness_score": _avg("freshness_score"),
+#     }
+
+#     page_type_breakdown = dict(Counter(p.get("page_type", "unknown") for p in page_results))
+
+#     issues, strengths, unknowns = _build_issues_and_strengths(page_results)
+
+#     pages_summary = [
+#         {
+#             "url": p["url"],
+#             "title": p["title"],
+#             **p["ai_visibility"],
+#         }
+#         for p in page_results
+#     ]
+
+#     return {
+#         "status": "success",
+#         "url": base_url,
+#         "pages_analyzed": len(page_results),
+#         "page_type_breakdown": page_type_breakdown,
+#         "target_query": target_query,
+#         "ai_readiness_score": overall_score,
+#         # legacy alias for existing frontend/API consumers
+#         "ai_visibility_score": overall_score,
+#         "actual_ai_visibility": {
+#             "status": "not_measured",
+#             "reason": "Real AI-query citation/mention data is not available to this crawler. This score reflects readiness proxies only, not observed visibility.",
+#         },
+#         "category_scores": category_averages,
+#         "issues": issues,
+#         "strengths": strengths,
+#         "unknown_metrics": unknowns,
+#         "results_preview": pages_summary,
+#     }