code_commenter.py

import re
from bs4 import BeautifulSoup, Comment

# Semantic tags to consider for HTML commenting
_SEMANTIC_TAGS = {"header", "nav", "main", "section", "article", "footer", "form"}

# Educational pedagogy for HTML tags
_TAG_PEDAGOGY = {
    "html": "root element — the top-level container for the entire web page",
    "head": "metadata container — holds information ABOUT the page (title, styles, scripts) that isn't visible",
    "body": "main content area — contains all the visible elements you see on the screen",
    "header": "page header — identifies the introductory section of a page or a section (often contains logos and navigation)",
    "nav": "navigation — contains links to help users move between pages or sections of the site",
    "main": "primary content — the central, unique content that defines this specific page",
    "section": "section — represents a standalone area of content, often with its own heading",
    "article": "article — a self-contained composition (like a blog post or news story) that makes sense on its own",
    "footer": "page footer — contains concluding information (copyright, contact info, related links) at the bottom",
    "form": "input form — a section for users to enter and submit information",
    "div": "division — a generic container used to group and organize other elements for layout",
    "p": "paragraph — defines a block of text, usually separated by a margin",
    "span": "inline container — used to style small snippets of text or elements within a line",
    "h1": "main heading — defines the most important title of the page",
    "h2": "sub-heading — divides content into major secondary sections",
    "h3": "sub-heading — divides content into smaller tertiary areas",
    "h4": "sub-heading — further level of content organization",
    "h5": "sub-heading — further level of content organization",
    "h6": "sub-heading — the lowest level of heading hierarchy",
    "ul": "unordered list — a list of items displayed with bullet points",
    "ol": "ordered list — a numbered list of items showing a specific sequence",
    "li": "list item — a single entry inside a bulleted or numbered list",
    "button": "interactive button — used to trigger actions (like opening a menu or submitting a form)",
    "img": "image — displays a visual graphic or photo on the page",
    "a": "hyperlink — creates a clickable link that connects to another page or section",
    "input": "data input — a field where users can type text or select options",
    "textarea": "text area — a multi-line box for entering larger amounts of text",
    "label": "input label — provides a descriptive name for a specific form field",
    "select": "dropdown menu — allows users to pick one or more options from a list",
    "table": "data table — used to organize complex information into rows and columns",
    "script": "logic script — contains JavaScript commands that make the page interactive",
    "style": "style rules — contains CSS code to control the appearance of the page",
    "aside": "sidebar content — represents content indirectly related to the main area (like ads or related links)",
    "video": "video player — embeds movie or animation content on the page",
    "audio": "audio player — embeds sound or music content on the page",
    "canvas": "drawing area — used for rendering graphics, games, or charts via JavaScript"
}

def _get_tag_representation(tag):
    """Return an opening tag with its attributes for inference. Truncate class list."""
    opening_tag = f"<{tag.name}"
    for attr, val in tag.attrs.items():
        if isinstance(val, list):
            val = " ".join(val)
            
        # Truncate class to max 3 items for CodeT5 token footprint
        if attr == "class":
            classes = val.split()
            if len(classes) > 3:
                val = " ".join(classes[:3]) + " ..."
                
        opening_tag += f' {attr}="{val}"'
    opening_tag += ">"
    
    return opening_tag

import sys
import os
sys.path.append(os.path.dirname(__file__))
from utils.class_explainer import explain_classes

def _annotate_html(soup, codet5_pipeline, comment_level):
    """
    Annotates structural HTML tags using three layers: Structural, Classes, Events.
    """
    tags_to_evaluate = soup.find_all(True)
    
    for tag in tags_to_evaluate:
        # 1. Determine if this tag is worth annotating
        event_attrs = ['onclick', 'onsubmit', 'onchange', 'oninput', 'onload', 'onkeydown', 'onmouseover']
        has_event = any(tag.get(ea) for ea in event_attrs)
        is_educational_target = tag.name in _TAG_PEDAGOGY
        has_id_or_class = tag.get("id") or tag.get("class")
        
        # Skip minor utility tags (unless they have special attrs) to avoid noise
        if not (is_educational_target or has_id_or_class or has_event):
            continue
            
        role = tag.name.upper()
        
        # 2. LAYER A: Structural/Educational Explanation
        structural_parts = []
        
        # A1. Start with the plain-English role from pedagogy dictionary
        pedagogy = _TAG_PEDAGOGY.get(tag.name)
        if pedagogy:
            structural_parts.append(pedagogy)
            
        # A2. Specialized attribute-level deterministic logic (replaces previous separate loops)
        if tag.name == 'img':
            src = tag.get('src', '')
            alt = tag.get('alt', '')
            if src:
                structural_parts.append(f"source: {src if len(src) <= 60 else src[:57] + '...'}")
            if alt:
                structural_parts.append(f"alt text: \"{alt}\"")
            elif not alt:
                structural_parts.append("no alt text (needs alt=\"description\" for accessibility)")
            if tag.get('width'): structural_parts.append(f"width: {tag.get('width')}px")
            
        elif tag.name == 'a' and tag.get('href'):
            href = tag.get('href', '#')
            structural_parts.append(f"links to: {href}")
            if tag.get('target') == '_blank': structural_parts.append("opens in new tab")
            
        elif tag.name == 'input':
            structural_parts.append(f"type: {tag.get('type', 'text')}")
            if tag.get('placeholder'): structural_parts.append(f"placeholder: \"{tag.get('placeholder')}\"")
            
        elif tag.name == 'script' and tag.get('src'):
            src = tag.get('src', '')
            lib = 'Tailwind CSS' if 'tailwind' in src else 'Bootstrap' if 'bootstrap' in src else 'jQuery' if 'jquery' in src else 'React' if 'react' in src else 'external JS'
            structural_parts.append(f"loads {lib} from CDN")

        # A3. Semantic Summarization (AI Model)
        # Use CodeT5 to describe the PURPOSE based on content or ID if it's a major container
        final_summary = ""
        should_run_model = (tag.name in _SEMANTIC_TAGS or tag.get("id")) and codet5_pipeline is not None and comment_level != "concise"
        
        if should_run_model:
            tag_repr = _get_tag_representation(tag)
            prompt = f"summarize: {tag_repr}"
            try:
                result = codet5_pipeline(prompt, max_new_tokens=50, do_sample=False)
                gen_text = result[0]['generated_text'].strip()
                # Reject garbage
                if '<' not in gen_text and '>' not in gen_text and len(gen_text.split()) >= 3 and not gen_text.lower().startswith(tag.name):
                    final_summary = gen_text
            except Exception:
                pass
        
        # A4. Fallback/Indicator string (e.g. div#hero)
        indicator = ""
        if tag.get("id"): indicator = f"#{tag.get('id')}"
        elif tag.get("class"):
            c = tag.get("class")
            first = c[0] if isinstance(c, list) else c.split()[0]
            indicator = f".{first}"
        
        indicator_str = f"{tag.name}{indicator}"
        
        # Combine everything into the structural comment
        full_structural_comment = f" {role}: {indicator_str} "
        if structural_parts:
            full_structural_comment += "— " + " — ".join(structural_parts)
        if final_summary:
            full_structural_comment += f" — ({final_summary})"

        tag.insert_before(soup.new_string("\n"))
        tag.insert_before(Comment(full_structural_comment))

        # 3. LAYER B: Classes
        if tag.get("class"):
            c_val = tag.get("class")
            c_str = " ".join(c_val) if isinstance(c_val, list) else c_val
            explained = explain_classes(c_str)
            if explained:
                tag.insert_before(soup.new_string("\n"))
                if "\n<!-- WARNING:" in explained:
                    parts = explained.split("\n<!-- WARNING:")
                    tag.insert_before(Comment(f" CLASSES: {parts[0]} "))
                    tag.insert_before(soup.new_string("\n"))
                    tag.insert_before(Comment(f" WARNING:{parts[1].replace('-->', '').strip()} "))
                else:
                    tag.insert_before(Comment(f" CLASSES: {explained} "))
            
        # 4. LAYER C: Events
        for ea in event_attrs:
            if tag.get(ea):
                tag.insert_before(soup.new_string("\n"))
                tag.insert_before(Comment(f" EVENT: triggers {tag.get(ea)} on {ea[2:]} "))


def _annotate_css(soup, comment_level):
    """
    Rule-based deterministic CSS comments.
    """
    for style_tag in soup.find_all("style"):
        css = style_tag.string or ""
        if not css.strip():
            continue
            
        header = (
            "/* ════════════════════════════════════════\n"
            "   CSS STYLES — Cascading Style Sheets\n"
            "   Styles defined here cascade down to all\n"
            "   matching child elements on the page.\n"
            "   ════════════════════════════════════════ */\n\n"
        )
        
        output = header
        
        # Dictionary: CSS property keyword → plain English explanation
        CSS_PROP_COMMENTS = {
            'margin':           'removes default spacing around elements',
            'margin-top':       'spacing above the element',
            'margin-bottom':    'spacing below the element',
            'margin-left':      'spacing to the left of the element',
            'margin-right':     'spacing to the right of the element',
            'padding':          'inner spacing inside the element',
            'padding-top':      'inner spacing above the content',
            'padding-bottom':   'inner spacing below the content',
            'padding-left':     'inner spacing on the left side',
            'padding-right':    'inner spacing on the right side',
            'display':          'controls how the element is rendered in the layout',
            'flex-direction':   'sets the direction of flex items (row or column)',
            'justify-content':  'aligns flex items along the main axis',
            'align-items':      'aligns flex items along the cross axis',
            'flex-wrap':        'allows flex items to wrap to the next line',
            'gap':              'spacing between grid or flex children',
            'position':         'controls how the element is positioned in the document',
            'top':              'distance from the top of the positioned parent',
            'bottom':           'distance from the bottom of the positioned parent',
            'left':             'distance from the left of the positioned parent',
            'right':            'distance from the right of the positioned parent',
            'z-index':          'controls stacking order — higher values appear on top',
            'width':            'horizontal size of the element',
            'height':           'vertical size of the element',
            'max-width':        'maximum width — prevents growing beyond this size',
            'min-width':        'minimum width — prevents shrinking below this size',
            'max-height':       'maximum height — prevents growing beyond this size',
            'min-height':       'minimum height — prevents shrinking below this size',
            'background':       'sets the background color or image',
            'background-color': 'fills the element background with a solid color',
            'background-image': 'sets an image as the background',
            'color':            'sets the text color',
            'font-size':        'controls the size of the text',
            'font-weight':      'controls how bold or thin the text appears',
            'font-family':      'sets the typeface (font) used for this element',
            'line-height':      'spacing between lines of text',
            'text-align':       'aligns text horizontally inside the element',
            'text-decoration':  'adds underline, strikethrough, or other text decorations',
            'letter-spacing':   'horizontal spacing between individual characters',
            'border':           'draws a visible border around the element',
            'border-radius':    'rounds the corners of the element',
            'outline':          'similar to border but does not affect layout',
            'box-shadow':       'adds a shadow effect behind the element',
            'opacity':          'controls element transparency (0=invisible, 1=fully visible)',
            'overflow':         'controls what happens when content overflows the element',
            'cursor':           'changes the look of the mouse cursor on hover',
            'transition':       'creates smooth animated changes when CSS properties change',
            'transform':        'applies 2D/3D transformation (rotate, scale, translate)',
            'animation':        'runs a CSS keyframe animation on this element',
            'grid-template-columns': 'defines the column structure of the grid',
            'grid-template-rows':    'defines the row structure of the grid',
            'object-fit':       'controls how content (image/video) fits inside the element',
            'pointer-events':   'controls whether the element responds to click events',
            'white-space':      'controls how text whitespace is handled',
            'content':          'inserts content before/after the element (used with ::before/::after)',
        }

        if comment_level in ["detailed", "educational"]:
            lines = css.split('\n')
            for line in lines:
                stripped = line.strip()
                if not stripped:
                    output += line + "\n"
                    continue

                # Selectors — annotate class/id/universal selectors
                if "{" in stripped and not stripped.startswith("@media"):
                    m = re.match(r'([\w\s.,#:\-\[\]="\'()*>~+^$]+)\{', stripped)
                    if m:
                        selectors = m.group(1).strip().split(',')
                        for sel in selectors:
                            sel = sel.strip()
                            if sel == '*':
                                output += "/* Universal selector: applies to EVERY element on the page */\n"
                            elif sel.startswith('.'):
                                output += f"/* Class: {sel} — Styling for {sel} component */\n"
                            elif sel.startswith('#'):
                                output += f"/* ID: {sel} — Styling for the #{sel.lstrip('#')} element */\n"
                            elif sel in ('body', 'html', 'header', 'footer', 'nav', 'main', 'section', 'article', 'aside'):
                                output += f"/* Element: <{sel}> — base styles for the {sel} element */\n"
                    output += line + "\n"
                    continue

                # Media queries
                if stripped.startswith("@media"):
                    bp_match = re.search(r'\((.*?)\)', stripped)
                    bp = bp_match.group(1) if bp_match else "specific screen size"
                    if "{" in line:
                        parts = line.split("{", 1)
                        output += parts[0] + "{\n  /* Responsive: styles below only apply when " + bp + " */\n" + (parts[1] if parts[1].strip() else "") + "\n"
                    else:
                        output += line + "\n"
                        output += f"  /* Responsive: styles below only apply when {bp} */\n"
                    continue

                # CSS property inline comments
                prop_match = re.match(r'\s*([\w-]+)\s*:', stripped)
                if prop_match and not stripped.startswith('/*') and not stripped.startswith('//'):
                    prop = prop_match.group(1).lower()
                    comment = CSS_PROP_COMMENTS.get(prop)
                    if comment:
                        indent = len(line) - len(line.lstrip())
                        output += ' ' * indent + f"/* {prop}: {comment} */\n"

                output += line + "\n"
        else:
            output += css
            
        style_tag.string = output


def _annotate_js(soup, comment_level):
    """
    Rule-based deterministic JS comments.
    """
    for script in soup.find_all("script"):
        if script.get("src"):
            continue
        js = script.string or ""
        if not js.strip():
            continue
            
        header = "// ═══ JAVASCRIPT — Interactive Logic ═══\n\n"
        output = header
        
        lines = js.split("\n")
        
        for line in lines:
            if comment_level in ["detailed", "educational"]:
                stripped = line.strip()
                
                # Fetch API calls
                fetch_match = re.search(r"fetch\(['\"]([^'\"]+)['\"]", stripped)
                if fetch_match:
                    url = fetch_match.group(1)
                    output += f"// API call: sends a network request to {url}\n"
                    
                # Variable DOM Bindings
                var_match = re.search(r"(?:const|let|var)\s+([a-zA-Z0-9_]+)\s*=\s*document\.(getElementById|querySelector)\(['\"]([^'\"]+)['\"]\)", stripped)
                if var_match:
                    var_name = var_match.group(1)
                    sel_type = var_match.group(2)
                    sel_val = var_match.group(3)
                    output += f"// Variable: stores a reference to the {sel_val} element for later use\n"
                    
                # DOM queries independently (if not caught by variable matcher above)
                elif "document.getElementById(" in stripped:
                    m = re.search(r"document\.getElementById\(['\"]([^'\"]+)['\"]\)", stripped)
                    if m: output += f"// DOM query: selects the element with id=\"{m.group(1)}\" from the page\n"
                elif "document.querySelectorAll(" in stripped:
                    m = re.search(r"document\.querySelectorAll\(['\"]([^'\"]+)['\"]\)", stripped)
                    if m: output += f"// DOM query: selects all elements matching \"{m.group(1)}\" as a list\n"
                elif "document.querySelector(" in stripped:
                    m = re.search(r"document\.querySelector\(['\"]([^'\"]+)['\"]\)", stripped)
                    if m: output += f"// DOM query: selects the first element matching \"{m.group(1)}\"\n"
                    
                # Event Listeners
                event_match = re.search(r"addEventListener\(['\"]([^'\"]+)['\"]", stripped)
                if event_match:
                    event = event_match.group(1)
                    output += f"// Event listener: Handles {event} event\n"
                elif "onclick=" in stripped.replace(" ", "") or ".onclick" in stripped:
                    output += f"// Event listener: Handles click event\n"
                
                # Function declarations
                func_match = re.match(r"function\s+([a-zA-Z0-9_]+)\s*\(", stripped)
                if func_match:
                    name = func_match.group(1)
                    output += f"// Function: {name} — handles {name} logic\n"
                else:
                    const_func_match = re.match(r"(?:const|let|var)\s+([a-zA-Z0-9_]+)\s*=\s*(?:function|async\s+function|\([^)]*\)\s*=>)", stripped)
                    if const_func_match:
                        name = const_func_match.group(1)
                        output += f"// Function: {name} — handles {name} logic\n"
                        
            output += line + "\n"
            
        script.string = output


def add_comments_to_webcraft_file(
    file_path=None,
    html_content=None,
    comment_level="concise",
    force_title=None,
    codet5_pipeline=None,
    dry_run=False,
    output_path=None
):
    """
    Agent 1: Inject semantic code comments.
    
    Can accept either a file path or raw HTML content.
    Returns: The annotated HTML string, and writes to output_path if not dry_run.
    """
    if not html_content and file_path:
        for enc in ["utf-8", "latin-1", "cp1252", "iso-8859-1"]:
            try:
                with open(file_path, "r", encoding=enc) as f:
                    html_content = f.read()
                break
            except Exception:
                continue
        if not html_content:
            with open(file_path, "rb") as f:
                html_content = f.read().decode("utf-8", errors="ignore")
                
    soup = BeautifulSoup(html_content, "html.parser")
    
    # 1. Annotate HTML Tags using CodeT5
    _annotate_html(soup, codet5_pipeline, comment_level)
    
    # 2. Annotate CSS
    _annotate_css(soup, comment_level)
    
    # 3. Annotate JS
    _annotate_js(soup, comment_level)
    
    if force_title:
        title_tag = soup.find("title")
        if title_tag:
            title_tag.string = force_title
        else:
            head = soup.find("head")
            if head:
                new_title = soup.new_tag("title")
                new_title.string = force_title
                head.insert(0, new_title)
                
    annotated_html = str(soup)
    
    if not dry_run:
        out_path = output_path or file_path
        if out_path:
            with open(out_path, "w", encoding="utf-8") as f:
                f.write(annotated_html)
                
    return annotated_html