code_commenter_improved.py

"""
WebCraft AI Annotation System — Improved Edition
================================================

This module implements the CodeT5-small-driven pedagogical annotator used by
the WebCraft backend to scaffold the transition from visual block-based
editing to textual coding for K-12 learners.

Annotation pipeline overview
----------------------------
Given an HTML document (with embedded <style> and <script> blocks), the
annotator parses the DOM with BeautifulSoup4 and walks every candidate node
to decide whether it is worth annotating. Three layers of commentary are then
composed:

    LAYER A - Structural / pedagogical
        A plain-English sentence describing the element's role drawn from a
        curated pedagogy dictionary (_TAG_PEDAGOGY), enriched with
        attribute-aware details (img src/alt, a href, input type, etc).
        For semantically important tags the CodeT5-small pipeline is invoked
        to produce a short, context-aware abstractive summary. All model
        outputs are filtered by a hallucination-guard heuristic that rejects
        answers that echo the tag name, return malformed angle-bracket tokens,
        or fall below a minimum word count.

    LAYER B - Classes
        Tailwind/Bootstrap-style utility classes are expanded into English by
        the utils.class_explainer helper so learners can read the visual
        effect of each class without consulting external documentation.

    LAYER C - Events
        Inline event-handler attributes (onclick, onsubmit, ...) are captured
        and rewritten as human-readable "triggers X on Y" comments so the
        behavioural surface of the page is visible in the source.

Comment levels
--------------
    concise     - Structural comments only, no AI inference (fastest path).
    detailed    - AI-enhanced: CodeT5 is invoked for semantic tags and for
                  the full CSS body. Every JS event listener, DOM query,
                  fetch() call, and if/else block is explained.
    educational - Same as `detailed` PLUS section-header banners prepended
                  to the HTML, CSS, and JS blocks that describe the role of
                  each code modality in the overall page. These banners are
                  designed to give a first-time reader a mental model of
                  what each section is for before they encounter specific
                  annotations.

CodeT5-small is used in a zero-shot configuration via the Hugging Face
text2text-generation pipeline. No fine-tuning is performed. The annotator is
pipeline-agnostic - if the `codet5_pipeline` argument is None, the system
silently degrades to deterministic rule-based output, keeping the backend
usable on resource-constrained servers or during cold-starts.

This file is intentionally importable side-effect-free and backward-compatible
with the production `code_commenter.py` API:
    add_comments_to_webcraft_file(
        file_path, html_content, comment_level,
        force_title, codet5_pipeline, dry_run, output_path
    )
"""

import logging
import re
import sys
import os
from bs4 import BeautifulSoup, Comment

logging.basicConfig(level=logging.WARNING)
logger = logging.getLogger(__name__)

sys.path.append(os.path.dirname(__file__))
try:
    from utils.class_explainer import explain_classes
except Exception:
    def explain_classes(_):
        return ""


_SEMANTIC_TAGS = {
    "header", "nav", "main", "section", "article", 
    "footer", "form", "h1", "h2", "h3"
}

_TAG_PEDAGOGY = {
    "html": "root element - the top-level container for the entire web page",
    "head": "metadata container - holds information ABOUT the page (title, styles, scripts) that isn't visible",
    "body": "main content area - contains all the visible elements you see on the screen",
    "header": "page header - identifies the introductory section of a page or a section (often contains logos and navigation)",
    "nav": "navigation - contains links to help users move between pages or sections of the site",
    "main": "primary content - the central, unique content that defines this specific page",
    "section": "section - represents a standalone area of content, often with its own heading",
    "article": "article - a self-contained composition (like a blog post or news story) that makes sense on its own",
    "footer": "page footer - contains concluding information (copyright, contact info, related links) at the bottom",
    "form": "input form - a section for users to enter and submit information",
    "div": "division - a generic container used to group and organize other elements for layout",
    "p": "paragraph - defines a block of text, usually separated by a margin",
    "span": "inline container - used to style small snippets of text or elements within a line",
    "h1": "main heading - defines the most important title of the page",
    "h2": "sub-heading - divides content into major secondary sections",
    "h3": "sub-heading - divides content into smaller tertiary areas",
    "h4": "sub-heading - further level of content organization",
    "h5": "sub-heading - further level of content organization",
    "h6": "sub-heading - the lowest level of heading hierarchy",
    "ul": "unordered list - a list of items displayed with bullet points",
    "ol": "ordered list - a numbered list of items showing a specific sequence",
    "li": "list item - a single entry inside a bulleted or numbered list",
    "button": "interactive button - used to trigger actions (like opening a menu or submitting a form)",
    "img": "image - displays a visual graphic or photo on the page",
    "a": "hyperlink - creates a clickable link that connects to another page or section",
    "input": "data input - a field where users can type text or select options",
    "textarea": "text area - a multi-line box for entering larger amounts of text",
    "label": "input label - provides a descriptive name for a specific form field",
    "select": "dropdown menu - allows users to pick one or more options from a list",
    "table": "data table - used to organize complex information into rows and columns",
    "script": "logic script - contains JavaScript commands that make the page interactive",
    "style": "style rules - contains CSS code to control the appearance of the page",
    "aside": "sidebar content - represents content indirectly related to the main area (like ads or related links)",
    "video": "video player - embeds movie or animation content on the page",
    "audio": "audio player - embeds sound or music content on the page",
    "canvas": "drawing area - used for rendering graphics, games, or charts via JavaScript",
}


# Cap AI output at ~20 words for pedagogical comments - longer summaries
# tend to drift into verbose, generic filler.
_MAX_AI_COMMENT_WORDS = 20


_HTML_SECTION_HEADER = (
    "\n<!--\n"
    "  === HTML STRUCTURE ===\n"
    "  HTML defines the skeleton of your page - every element you see\n"
    "  is described here. The browser reads this top to bottom and builds\n"
    "  the page layout. Without HTML there is nothing to display.\n"
    "  =================================================================\n"
    "-->\n"
)

_CSS_SECTION_HEADER = (
    "/* =====================================================================\n"
    "   CSS STYLES - Cascading Style Sheets\n"
    "   CSS is the rulebook for how your HTML elements LOOK. Colors, sizes,\n"
    "   spacing, fonts, and layout all live here. Rules cascade from parent\n"
    "   elements to their children unless overridden.\n"
    "   ===================================================================== */\n\n"
)

_JS_SECTION_HEADER = (
    "// =====================================================================\n"
    "// JAVASCRIPT - Interactive Behaviour\n"
    "// JavaScript makes the page RESPOND to the user. It listens for\n"
    "// clicks, typing, and page-load events, and changes the HTML/CSS\n"
    "// live. Without JS a page is static; with JS it is an application.\n"
    "// =====================================================================\n\n"
)


def _sanitize_ai_text(text, tag_name=None):
    """Reject obviously broken or over-long CodeT5 outputs."""
    if not text:
        return ""
    cleaned = text.strip().strip('"').strip("'")
    if not cleaned:
        return ""
    if "<" in cleaned or ">" in cleaned:
        return ""
    words = cleaned.split()
    if len(words) < 3:
        return ""
    # Only reject if the first word IS the tag and the rest is too short
    if tag_name and words[0].lower() == tag_name.lower() and len(words) < 5:
        return ""
    if len(words) > _MAX_AI_COMMENT_WORDS:
        cleaned = " ".join(words[:_MAX_AI_COMMENT_WORDS]).rstrip(",;") + "..."
    return cleaned


def _invoke_ai(codet5_pipeline, prompt, max_new_tokens=60):
    """Safe wrapper around the HuggingFace text2text-generation pipeline."""
    if codet5_pipeline is None:
        return ""
    try:
        result = codet5_pipeline(prompt, max_new_tokens=max_new_tokens, do_sample=False)
        if result and isinstance(result, list):
            return result[0].get("generated_text", "").strip()
    except Exception:
        return ""
    return ""


def _invoke_ai_batch(codet5_pipeline, prompts, max_new_tokens=100, batch_size=8):
    """Efficiently run multiple prompts through the transformer pipeline in a single batch."""
    if codet5_pipeline is None or not prompts:
        return [""] * len(prompts)
    
    try:
        # Pipeline handles list of strings and batching automatically
        results = codet5_pipeline(
            prompts, 
            max_new_tokens=max_new_tokens, 
            do_sample=False,
            batch_size=batch_size
        )
        
        outputs = []
        for res in results:
            if isinstance(res, list):
                outputs.append(res[0].get("generated_text", "").strip())
            else:
                outputs.append(res.get("generated_text", "").strip())
        return outputs
    except Exception as e:
        logger.warning(f"Batch AI inference failed: {e}")
        return [""] * len(prompts)



def _truncate_to_tokens(text: str, max_tokens: int = 400) -> str:
    """Approximate token limit by word count (1 token ~ 0.75 words)."""
    words = text.split()
    limit = int(max_tokens * 0.75)
    return " ".join(words[:limit])


_LANDMARK_TAGS = ["header", "nav", "main", "section", "article", "aside", "footer"]


def _chunk_by_landmarks(soup):
    """Yield (tag, text) tuples for each landmark section in the page."""
    found = []
    for name in _LANDMARK_TAGS:
        for tag in soup.find_all(name):
            found.append((tag, _strip_class_attrs(tag)))
    if not found:
        body = soup.find("body")
        if body:
            found.append((body, _strip_class_attrs(body)))
    return found


def _strip_class_attrs(tag) -> str:
    """Return tag HTML with all class= attributes removed for cleaner T5 input."""
    from copy import copy
    clone = copy(tag)
    if hasattr(clone, 'attrs') and 'class' in clone.attrs:
        del clone.attrs['class']
    opening = f"<{clone.name}"
    for attr, val in clone.attrs.items():
        if isinstance(val, list):
            val = " ".join(val)
        opening += f' {attr}="{val}"'
    opening += ">"
    text = tag.get_text(separator=" ", strip=True)
    if len(text) > 200:
        text = text[:200] + "..."
    return f"{opening} {text}"


def _strip_existing_ai_comments(soup):
    """
    Remove any existing AI-generated comments from the HTML, CSS, and JS.
    This prevents duplicates when the commenter is run multiple times.
    """
    # 1. Strip HTML comments matching WebCraft patterns
    html_patterns = [
        # Matches " ROLE: indicator - pedagogy - (summary)"
        r"^\s*(HTML|BODY|HEAD|HEADER|NAV|MAIN|SECTION|ARTICLE|FOOTER|FORM|DIV|P|SPAN|H[1-6]|UL|OL|LI|BUTTON|IMG|A|INPUT|TEXTAREA|LABEL|SELECT|TABLE|SCRIPT|STYLE|ASIDE|VIDEO|AUDIO|CANVAS):.*",
        r"^\s*CLASSES:.*",
        r"^\s*EVENT:.*",
        r"^\s*WARNING:.*",
        r".*=== HTML STRUCTURE ===.*",
        r".*CSS STYLES - Cascading Style Sheets.*",
        r".*JAVASCRIPT - Interactive Behaviour.*",
        r".*JAVASCRIPT - Interactive Logic.*"
    ]
    html_regex = re.compile("|".join(html_patterns), re.IGNORECASE)

    for comment in soup.find_all(string=lambda text: isinstance(text, Comment)):
        if html_regex.match(comment):
            # Also remove the newline before the comment if it's just whitespace
            prev_sibling = comment.previous_sibling
            if prev_sibling and isinstance(prev_sibling, str) and not prev_sibling.strip():
                prev_sibling.extract()
            comment.extract()

    # 2. Strip CSS comments from <style> tags
    css_overview_pattern = re.compile(r'/\* AI OVERVIEW: .*? \*/\s*', re.DOTALL)
    css_section_pattern = re.compile(r'/\* =+ CSS STYLES .*? =+ \*/\s*', re.DOTALL)
    css_selector_pattern = re.compile(r'/\* (Universal selector|Class|ID|Element|Responsive): .*? \*/\s*')
    
    # Precise property comment matching using CSS_PROP_COMMENTS
    prop_patterns = []
    for prop, desc in CSS_PROP_COMMENTS.items():
        prop_patterns.append(re.escape(f"/* {prop}: {desc} */"))
    prop_regex = re.compile("|".join(prop_patterns), re.IGNORECASE)

    for style in soup.find_all("style"):
        if not style.string:
            continue
        content = style.string
        content = css_overview_pattern.sub("", content)
        content = css_section_pattern.sub("", content)
        content = css_selector_pattern.sub("", content)
        
        # Remove property comments line-by-line to stay safe
        lines = content.split("\n")
        new_lines = []
        for line in lines:
            if not prop_regex.search(line):
                new_lines.append(line)
        style.string = "\n".join(new_lines).strip()

    # 3. Strip JS comments from <script> tags
    js_section_pattern = re.compile(r'// =+ JAVASCRIPT .*? =+\s*', re.DOTALL)
    js_prefix_pattern = re.compile(r'// (Fetch|Variable|DOM query|Event listener|Branch|Function):? .*')
    js_func_pattern = re.compile(r'// Function `.*?` - .*')

    for script in soup.find_all("script"):
        if script.get("src") or not script.string:
            continue
        content = script.string
        content = js_section_pattern.sub("", content)
        
        lines = content.split("\n")
        new_lines = []
        for line in lines:
            if not (js_prefix_pattern.match(line.strip()) or js_func_pattern.match(line.strip())):
                new_lines.append(line)
        script.string = "\n".join(new_lines).strip()


def _annotate_html(soup, codet5_pipeline, comment_level):
    """Three-layer HTML annotator: Structural / Classes / Events.
    Uses batch inference for performance.
    """
    event_attrs = [
        "onclick", "onsubmit", "onchange", "oninput",
        "onload", "onkeydown", "onmouseover",
    ]

    # PASS 1: Identify targets and collect prompts
    targets = []
    prompts = []

    
    for tag in soup.find_all(True):
        has_event = any(tag.get(ea) for ea in event_attrs)
        is_educational_target = tag.name in _TAG_PEDAGOGY
        has_id_or_class = tag.get("id") or tag.get("class")

        if not (is_educational_target or has_id_or_class or has_event):
            continue

        should_run_ai = (
            (tag.name in _SEMANTIC_TAGS or tag.get("id"))
            and codet5_pipeline is not None
            and comment_level in ("detailed", "educational")
        )

        prompt = None
        if should_run_ai:
            prompt = (
                f"Explain this web element for a beginner: WHAT it does, "
                f"WHY it is placed here in context, and what would "
                f"BREAK or happen to the visitor if it were removed. "
                f"IMPORTANT: DO NOT start your answer with the word '{tag.name}'. "
                f"Context: {_strip_class_attrs(tag)}"
            )
            prompts.append(prompt)
        
        targets.append({
            "tag": tag,
            "has_event": has_event,
            "prompt_idx": len(prompts) - 1 if prompt else -1
        })

    # BATCH AI INVOCATION
    ai_outputs = []
    if prompts:
        ai_outputs = _invoke_ai_batch(codet5_pipeline, prompts, max_new_tokens=100)

    # PASS 2: Apply annotations
    for target in targets:
        tag = target["tag"]
        role = tag.name.upper()
        structural_parts = []
        
        pedagogy = _TAG_PEDAGOGY.get(tag.name)
        if pedagogy:
            structural_parts.append(pedagogy)

        # Attribute-aware annotations
        if tag.name == "img":
            src = tag.get("src", "")
            alt = tag.get("alt", "")
            if src:
                structural_parts.append(f"source: {src if len(src) <= 60 else src[:57] + '...'}")
            if alt:
                structural_parts.append(f'alt text: "{alt}"')
            else:
                structural_parts.append('no alt text (needs alt="description" for accessibility)')
            if tag.get("width"):
                structural_parts.append(f"width: {tag.get('width')}px")
        elif tag.name == "a" and tag.get("href"):
            href = tag.get("href", "#")
            structural_parts.append(f"links to: {href}")
            if tag.get("target") == "_blank":
                structural_parts.append("opens in new tab")
        elif tag.name == "input":
            structural_parts.append(f"type: {tag.get('type', 'text')}")
            if tag.get("placeholder"):
                structural_parts.append(f'placeholder: "{tag.get("placeholder")}"')
        elif tag.name == "script" and tag.get("src"):
            src = tag.get("src", "")
            lib = (
                "Tailwind CSS" if "tailwind" in src
                else "Bootstrap" if "bootstrap" in src
                else "jQuery" if "jquery" in src
                else "React" if "react" in src
                else "external JS"
            )
            structural_parts.append(f"loads {lib} from CDN")

        # Apply AI summary
        final_summary = ""
        if target["prompt_idx"] != -1 and target["prompt_idx"] < len(ai_outputs):
            raw = ai_outputs[target["prompt_idx"]]
            final_summary = _sanitize_ai_text(raw, tag.name)

        # Construct final comment
        indicator = ""
        if tag.get("id"):
            indicator = f"#{tag.get('id')}"
        elif tag.get("class"):
            c = tag.get("class")
            first = c[0] if isinstance(c, list) else c.split()[0]
            indicator = f".{first}"
        indicator_str = f"{tag.name}{indicator}"

        full_structural_comment = f" {role}: {indicator_str} "
        if structural_parts:
            full_structural_comment += "- " + " - ".join(structural_parts)
        if final_summary:
            full_structural_comment += f" - ({final_summary})"

        tag.insert_before(soup.new_string("\n"))
        tag.insert_before(Comment(full_structural_comment))

        # Classes annotation
        if tag.get("class"):
            c_val = tag.get("class")
            c_str = " ".join(c_val) if isinstance(c_val, list) else c_val
            explained = explain_classes(c_str)
            if explained:
                tag.insert_before(soup.new_string("\n"))
                if "\n<!-- WARNING:" in explained:
                    parts = explained.split("\n<!-- WARNING:")
                    tag.insert_before(Comment(f" CLASSES: {parts[0]} "))
                    tag.insert_before(soup.new_string("\n"))
                    tag.insert_before(Comment(f" WARNING:{parts[1].replace('-->', '').strip()} "))
                else:
                    tag.insert_before(Comment(f" CLASSES: {explained} "))

        # Events annotation
        for ea in event_attrs:
            if tag.get(ea):
                tag.insert_before(soup.new_string("\n"))
                tag.insert_before(Comment(f" EVENT: triggers {tag.get(ea)} on {ea[2:]} "))



CSS_PROP_COMMENTS = {
    "margin": "removes default spacing around elements",
    "margin-top": "spacing above the element",
    "margin-bottom": "spacing below the element",
    "margin-left": "spacing to the left of the element",
    "margin-right": "spacing to the right of the element",
    "padding": "inner spacing inside the element",
    "padding-top": "inner spacing above the content",
    "padding-bottom": "inner spacing below the content",
    "padding-left": "inner spacing on the left side",
    "padding-right": "inner spacing on the right side",
    "display": "controls how the element is rendered in the layout",
    "flex-direction": "sets the direction of flex items (row or column)",
    "justify-content": "aligns flex items along the main axis",
    "align-items": "aligns flex items along the cross axis",
    "flex-wrap": "allows flex items to wrap to the next line",
    "gap": "spacing between grid or flex children",
    "position": "controls how the element is positioned in the document",
    "top": "distance from the top of the positioned parent",
    "bottom": "distance from the bottom of the positioned parent",
    "left": "distance from the left of the positioned parent",
    "right": "distance from the right of the positioned parent",
    "z-index": "controls stacking order - higher values appear on top",
    "width": "horizontal size of the element",
    "height": "vertical size of the element",
    "max-width": "maximum width - prevents growing beyond this size",
    "min-width": "minimum width - prevents shrinking below this size",
    "max-height": "maximum height - prevents growing beyond this size",
    "min-height": "minimum height - prevents shrinking below this size",
    "background": "sets the background color or image",
    "background-color": "fills the element background with a solid color",
    "background-image": "sets an image as the background",
    "color": "sets the text color",
    "font-size": "controls the size of the text",
    "font-weight": "controls how bold or thin the text appears",
    "font-family": "sets the typeface (font) used for this element",
    "line-height": "spacing between lines of text",
    "text-align": "aligns text horizontally inside the element",
    "text-decoration": "adds underline, strikethrough, or other text decorations",
    "letter-spacing": "horizontal spacing between individual characters",
    "border": "draws a visible border around the element",
    "border-radius": "rounds the corners of the element",
    "outline": "similar to border but does not affect layout",
    "box-shadow": "adds a shadow effect behind the element",
    "opacity": "controls element transparency (0=invisible, 1=fully visible)",
    "overflow": "controls what happens when content overflows the element",
    "cursor": "changes the look of the mouse cursor on hover",
    "transition": "creates smooth animated changes when CSS properties change",
    "transform": "applies 2D/3D transformation (rotate, scale, translate)",
    "animation": "runs a CSS keyframe animation on this element",
    "grid-template-columns": "defines the column structure of the grid",
    "grid-template-rows": "defines the row structure of the grid",
    "object-fit": "controls how content (image/video) fits inside the element",
    "pointer-events": "controls whether the element responds to click events",
    "white-space": "controls how text whitespace is handled",
    "content": "inserts content before/after the element (used with ::before/::after)",
}


def _annotate_css_with_ai(css_text, codet5_pipeline):
    """Invoke CodeT5 once over the entire CSS body and return a short
    natural-language summary to prepend to the stylesheet. The summary is
    wrapped as a CSS block comment. This is the only AI call made per
    stylesheet - the inline property comments remain rule-based."""
    if not css_text.strip() or codet5_pipeline is None:
        return ""
    clean_css = re.sub(r'\.[a-z][\w-]*\s*\{[^}]*\}', '', css_text)
    prompt = (
        "summarize this web stylesheet: "
        + _truncate_to_tokens(clean_css, max_tokens=400)
    )
    raw = _invoke_ai(codet5_pipeline, prompt, max_new_tokens=80)
    summary = _sanitize_ai_text(raw)
    if not summary:
        return ""
    return (
        "/* AI OVERVIEW: " + summary + " */\n\n"
    )


def _annotate_css(soup, codet5_pipeline, comment_level):
    """Rule-based + (optional) AI CSS commenter."""
    for style_tag in soup.find_all("style"):
        css = style_tag.string or ""
        if not css.strip():
            continue

        output = ""
        if comment_level == "educational":
            output += _CSS_SECTION_HEADER
        else:
            output += (
                "/* ==================================\n"
                "   CSS STYLES - Cascading Style Sheets\n"
                "   ================================== */\n\n"
            )

        if comment_level in ("detailed", "educational"):
            output += _annotate_css_with_ai(css, codet5_pipeline)

        if comment_level in ("detailed", "educational"):
            for line in css.split("\n"):
                stripped = line.strip()
                if not stripped:
                    output += line + "\n"
                    continue

                if "{" in stripped and not stripped.startswith("@media"):
                    m = re.match(r'([\w\s.,#:\-\[\]="\'()*>~+^$]+)\{', stripped)
                    if m:
                        for sel in m.group(1).strip().split(","):
                            sel = sel.strip()
                            if sel == "*":
                                output += "/* Universal selector: applies to EVERY element on the page */\n"
                            elif sel.startswith("."):
                                output += f"/* Class: {sel} - styles every element carrying this class */\n"
                            elif sel.startswith("#"):
                                output += f"/* ID: {sel} - styles the single element with this id */\n"
                            elif sel in (
                                "body", "html", "header", "footer",
                                "nav", "main", "section", "article", "aside",
                            ):
                                output += f"/* Element: <{sel}> - base styles for every {sel} element on the page */\n"
                    output += line + "\n"
                    continue

                if stripped.startswith("@media"):
                    bp_match = re.search(r"\((.*?)\)", stripped)
                    bp = bp_match.group(1) if bp_match else "specific screen size"
                    if "{" in line:
                        parts = line.split("{", 1)
                        output += (
                            parts[0]
                            + "{\n  /* Responsive: styles below only apply when "
                            + bp
                            + " */\n"
                            + (parts[1] if parts[1].strip() else "")
                            + "\n"
                        )
                    else:
                        output += line + "\n"
                        output += f"  /* Responsive: styles below only apply when {bp} */\n"
                    continue

                prop_match = re.match(r"\s*([\w-]+)\s*:", stripped)
                if (
                    prop_match
                    and not stripped.startswith("/*")
                    and not stripped.startswith("//")
                ):
                    prop = prop_match.group(1).lower()
                    comment = CSS_PROP_COMMENTS.get(prop)
                    if comment:
                        indent = len(line) - len(line.lstrip())
                        output += " " * indent + f"/* {prop}: {comment} */\n"

                output += line + "\n"
        else:
            output += css

        style_tag.string = output


_IF_ELSE_PATTERN = re.compile(r"^\s*(if|else\s+if|else)\s*(\([^)]*\))?")


def _annotate_js(soup, codet5_pipeline, comment_level):
    """Rule-based JS commenter with improved event / DOM / fetch / if-else
    detection. CodeT5 is optionally invoked once per function declaration in
    educational mode to produce a short human summary of what the function
    does - this mirrors the way a teacher would narrate the code line-by-line.
    """
    for script in soup.find_all("script"):
        if script.get("src"):
            continue
        js = script.string or ""
        if not js.strip():
            continue

        output = ""
        if comment_level == "educational":
            output += _JS_SECTION_HEADER
        else:
            output += "// === JAVASCRIPT - Interactive Logic ===\n\n"

        for line in js.split("\n"):
            if comment_level in ("detailed", "educational"):
                stripped = line.strip()

                fetch_match = re.search(r"fetch\(['\"]([^'\"]+)['\"]", stripped)
                if fetch_match:
                    url = fetch_match.group(1)
                    output += (
                        f"// Fetch: downloads data from {url} over the network "
                        f"(returns a Promise you await or .then on)\n"
                    )

                var_match = re.search(
                    r"(?:const|let|var)\s+([a-zA-Z0-9_]+)\s*=\s*"
                    r"document\.(getElementById|querySelector|querySelectorAll)\("
                    r"['\"]([^'\"]+)['\"]\)",
                    stripped,
                )
                if var_match:
                    var_name = var_match.group(1)
                    sel_val = var_match.group(3)
                    output += (
                        f"// Variable `{var_name}` stores a reference to the "
                        f"'{sel_val}' element for later use\n"
                    )
                elif "document.getElementById(" in stripped:
                    m = re.search(
                        r"document\.getElementById\(['\"]([^'\"]+)['\"]\)", stripped
                    )
                    if m:
                        output += (
                            f"// DOM query: selects the element with id=\"{m.group(1)}\" "
                            f"from the page\n"
                        )
                elif "document.querySelectorAll(" in stripped:
                    m = re.search(
                        r"document\.querySelectorAll\(['\"]([^'\"]+)['\"]\)", stripped
                    )
                    if m:
                        output += (
                            f"// DOM query: selects ALL elements matching \"{m.group(1)}\" "
                            f"as a list\n"
                        )
                elif "document.querySelector(" in stripped:
                    m = re.search(
                        r"document\.querySelector\(['\"]([^'\"]+)['\"]\)", stripped
                    )
                    if m:
                        output += (
                            f"// DOM query: selects the first element matching "
                            f"\"{m.group(1)}\"\n"
                        )

                event_match = re.search(
                    r"\.addEventListener\(['\"]([^'\"]+)['\"]", stripped
                )
                if event_match:
                    event = event_match.group(1)
                    pretty = {
                        "click": "the user clicks the element",
                        "submit": "the user submits the form",
                        "input": "the user types in the field",
                        "change": "the field value changes",
                        "mouseover": "the mouse moves over the element",
                        "keydown": "a key is pressed",
                        "load": "the page finishes loading",
                    }.get(event, f"the '{event}' event fires")
                    output += (
                        f"// Event listener: runs the following code whenever {pretty}\n"
                    )
                elif (
                    "onclick=" in stripped.replace(" ", "")
                    or ".onclick" in stripped
                ):
                    output += "// Event listener: runs when the element is clicked\n"

                if _IF_ELSE_PATTERN.match(stripped):
                    kw = _IF_ELSE_PATTERN.match(stripped).group(1)
                    if kw == "if":
                        output += (
                            "// Branch: the following code only runs when the "
                            "condition in parentheses is true\n"
                        )
                    elif kw.startswith("else if"):
                        output += (
                            "// Branch: this alternative runs only when the "
                            "previous conditions were false AND this one is true\n"
                        )
                    else:
                        output += (
                            "// Branch: fallback code - runs only if every "
                            "previous if/else-if condition was false\n"
                        )

                func_match = re.match(
                    r"function\s+([a-zA-Z0-9_]+)\s*\(", stripped
                )
                if func_match:
                    name = func_match.group(1)
                    summary = ""
                    if (
                        comment_level == "educational"
                        and codet5_pipeline is not None
                    ):
                        prompt = (
                            "summarize: explain in one short sentence what this "
                            "javascript function does: " + _truncate_to_tokens(stripped, max_tokens=200)
                        )
                        summary = _sanitize_ai_text(
                            _invoke_ai(codet5_pipeline, prompt, max_new_tokens=50)
                        )
                    if summary:
                        output += f"// Function `{name}` - {summary}\n"
                    else:
                        output += (
                            f"// Function `{name}` - declared here and callable "
                            f"elsewhere in the script\n"
                        )
                else:
                    const_func_match = re.match(
                        r"(?:const|let|var)\s+([a-zA-Z0-9_]+)\s*=\s*"
                        r"(?:function|async\s+function|\([^)]*\)\s*=>)",
                        stripped,
                    )
                    if const_func_match:
                        name = const_func_match.group(1)
                        output += (
                            f"// Function `{name}` - an arrow/function expression "
                            f"stored in a variable\n"
                        )

            output += line + "\n"

        script.string = output


def _prepend_educational_html_header(annotated_html):
    """Insert the HTML educational banner immediately after the opening
    <body> tag so the comment is the very first thing inside the rendered
    page source. Falls back to prepending at the top if no <body> exists."""
    body_match = re.search(r"<body[^>]*>", annotated_html, flags=re.IGNORECASE)
    if body_match:
        idx = body_match.end()
        return annotated_html[:idx] + _HTML_SECTION_HEADER + annotated_html[idx:]
    return _HTML_SECTION_HEADER + annotated_html


def add_comments_to_webcraft_file(
    file_path=None,
    html_content=None,
    comment_level="concise",
    force_title=None,
    codet5_pipeline=None,
    dry_run=False,
    output_path=None,
):
    """Agent 1: inject semantic code comments.

    Accepts either a file path or raw HTML content. Returns the annotated
    HTML string; writes to `output_path` (or back to `file_path`) unless
    `dry_run` is set. The `comment_level` argument accepts 'concise',
    'detailed', or 'educational'.
    """
    if not html_content and file_path:
        for enc in ("utf-8", "latin-1", "cp1252", "iso-8859-1"):
            try:
                with open(file_path, "r", encoding=enc) as f:
                    html_content = f.read()
                break
            except Exception:
                continue
        if not html_content:
            with open(file_path, "rb") as f:
                html_content = f.read().decode("utf-8", errors="ignore")

    soup = BeautifulSoup(html_content, "html.parser")

    # Clean up existing AI comments before adding new ones
    _strip_existing_ai_comments(soup)

    _annotate_html(soup, codet5_pipeline, comment_level)
    _annotate_css(soup, codet5_pipeline, comment_level)
    _annotate_js(soup, codet5_pipeline, comment_level)

    if force_title:
        title_tag = soup.find("title")
        if title_tag:
            title_tag.string = force_title
        else:
            head = soup.find("head")
            if head:
                new_title = soup.new_tag("title")
                new_title.string = force_title
                head.insert(0, new_title)

    annotated_html = str(soup)

    if comment_level == "educational":
        annotated_html = _prepend_educational_html_header(annotated_html)

    if not dry_run:
        out_path = output_path or file_path
        if out_path:
            with open(out_path, "w", encoding="utf-8") as f:
                f.write(annotated_html)

    return annotated_html