import re from bs4 import BeautifulSoup, Comment # Semantic tags to consider for HTML commenting _SEMANTIC_TAGS = {"header", "nav", "main", "section", "article", "footer", "form"} # Educational pedagogy for HTML tags _TAG_PEDAGOGY = { "html": "root element — the top-level container for the entire web page", "head": "metadata container — holds information ABOUT the page (title, styles, scripts) that isn't visible", "body": "main content area — contains all the visible elements you see on the screen", "header": "page header — identifies the introductory section of a page or a section (often contains logos and navigation)", "nav": "navigation — contains links to help users move between pages or sections of the site", "main": "primary content — the central, unique content that defines this specific page", "section": "section — represents a standalone area of content, often with its own heading", "article": "article — a self-contained composition (like a blog post or news story) that makes sense on its own", "footer": "page footer — contains concluding information (copyright, contact info, related links) at the bottom", "form": "input form — a section for users to enter and submit information", "div": "division — a generic container used to group and organize other elements for layout", "p": "paragraph — defines a block of text, usually separated by a margin", "span": "inline container — used to style small snippets of text or elements within a line", "h1": "main heading — defines the most important title of the page", "h2": "sub-heading — divides content into major secondary sections", "h3": "sub-heading — divides content into smaller tertiary areas", "h4": "sub-heading — further level of content organization", "h5": "sub-heading — further level of content organization", "h6": "sub-heading — the lowest level of heading hierarchy", "ul": "unordered list — a list of items displayed with bullet points", "ol": "ordered list — a numbered list of items showing a specific sequence", "li": "list item — a single entry inside a bulleted or numbered list", "button": "interactive button — used to trigger actions (like opening a menu or submitting a form)", "img": "image — displays a visual graphic or photo on the page", "a": "hyperlink — creates a clickable link that connects to another page or section", "input": "data input — a field where users can type text or select options", "textarea": "text area — a multi-line box for entering larger amounts of text", "label": "input label — provides a descriptive name for a specific form field", "select": "dropdown menu — allows users to pick one or more options from a list", "table": "data table — used to organize complex information into rows and columns", "script": "logic script — contains JavaScript commands that make the page interactive", "style": "style rules — contains CSS code to control the appearance of the page", "aside": "sidebar content — represents content indirectly related to the main area (like ads or related links)", "video": "video player — embeds movie or animation content on the page", "audio": "audio player — embeds sound or music content on the page", "canvas": "drawing area — used for rendering graphics, games, or charts via JavaScript" } def _get_tag_representation(tag): """Return an opening tag with its attributes for inference. Truncate class list.""" opening_tag = f"<{tag.name}" for attr, val in tag.attrs.items(): if isinstance(val, list): val = " ".join(val) # Truncate class to max 3 items for CodeT5 token footprint if attr == "class": classes = val.split() if len(classes) > 3: val = " ".join(classes[:3]) + " ..." opening_tag += f' {attr}="{val}"' opening_tag += ">" return opening_tag import sys import os sys.path.append(os.path.dirname(__file__)) from utils.class_explainer import explain_classes def _annotate_html(soup, codet5_pipeline, comment_level): """ Annotates structural HTML tags using three layers: Structural, Classes, Events. """ tags_to_evaluate = soup.find_all(True) for tag in tags_to_evaluate: # 1. Determine if this tag is worth annotating event_attrs = ['onclick', 'onsubmit', 'onchange', 'oninput', 'onload', 'onkeydown', 'onmouseover'] has_event = any(tag.get(ea) for ea in event_attrs) is_educational_target = tag.name in _TAG_PEDAGOGY has_id_or_class = tag.get("id") or tag.get("class") # Skip minor utility tags (unless they have special attrs) to avoid noise if not (is_educational_target or has_id_or_class or has_event): continue role = tag.name.upper() # 2. LAYER A: Structural/Educational Explanation structural_parts = [] # A1. Start with the plain-English role from pedagogy dictionary pedagogy = _TAG_PEDAGOGY.get(tag.name) if pedagogy: structural_parts.append(pedagogy) # A2. Specialized attribute-level deterministic logic (replaces previous separate loops) if tag.name == 'img': src = tag.get('src', '') alt = tag.get('alt', '') if src: structural_parts.append(f"source: {src if len(src) <= 60 else src[:57] + '...'}") if alt: structural_parts.append(f"alt text: \"{alt}\"") elif not alt: structural_parts.append("no alt text (needs alt=\"description\" for accessibility)") if tag.get('width'): structural_parts.append(f"width: {tag.get('width')}px") elif tag.name == 'a' and tag.get('href'): href = tag.get('href', '#') structural_parts.append(f"links to: {href}") if tag.get('target') == '_blank': structural_parts.append("opens in new tab") elif tag.name == 'input': structural_parts.append(f"type: {tag.get('type', 'text')}") if tag.get('placeholder'): structural_parts.append(f"placeholder: \"{tag.get('placeholder')}\"") elif tag.name == 'script' and tag.get('src'): src = tag.get('src', '') lib = 'Tailwind CSS' if 'tailwind' in src else 'Bootstrap' if 'bootstrap' in src else 'jQuery' if 'jquery' in src else 'React' if 'react' in src else 'external JS' structural_parts.append(f"loads {lib} from CDN") # A3. Semantic Summarization (AI Model) # Use CodeT5 to describe the PURPOSE based on content or ID if it's a major container final_summary = "" should_run_model = (tag.name in _SEMANTIC_TAGS or tag.get("id")) and codet5_pipeline is not None and comment_level != "concise" if should_run_model: tag_repr = _get_tag_representation(tag) prompt = f"summarize: {tag_repr}" try: result = codet5_pipeline(prompt, max_new_tokens=50, do_sample=False) gen_text = result[0]['generated_text'].strip() # Reject garbage if '<' not in gen_text and '>' not in gen_text and len(gen_text.split()) >= 3 and not gen_text.lower().startswith(tag.name): final_summary = gen_text except Exception: pass # A4. Fallback/Indicator string (e.g. div#hero) indicator = "" if tag.get("id"): indicator = f"#{tag.get('id')}" elif tag.get("class"): c = tag.get("class") first = c[0] if isinstance(c, list) else c.split()[0] indicator = f".{first}" indicator_str = f"{tag.name}{indicator}" # Combine everything into the structural comment full_structural_comment = f" {role}: {indicator_str} " if structural_parts: full_structural_comment += "— " + " — ".join(structural_parts) if final_summary: full_structural_comment += f" — ({final_summary})" tag.insert_before(soup.new_string("\n")) tag.insert_before(Comment(full_structural_comment)) # 3. LAYER B: Classes if tag.get("class"): c_val = tag.get("class") c_str = " ".join(c_val) if isinstance(c_val, list) else c_val explained = explain_classes(c_str) if explained: tag.insert_before(soup.new_string("\n")) if "\n', '').strip()} ")) else: tag.insert_before(Comment(f" CLASSES: {explained} ")) # 4. LAYER C: Events for ea in event_attrs: if tag.get(ea): tag.insert_before(soup.new_string("\n")) tag.insert_before(Comment(f" EVENT: triggers {tag.get(ea)} on {ea[2:]} ")) def _annotate_css(soup, comment_level): """ Rule-based deterministic CSS comments. """ for style_tag in soup.find_all("style"): css = style_tag.string or "" if not css.strip(): continue header = ( "/* ════════════════════════════════════════\n" " CSS STYLES — Cascading Style Sheets\n" " Styles defined here cascade down to all\n" " matching child elements on the page.\n" " ════════════════════════════════════════ */\n\n" ) output = header # Dictionary: CSS property keyword → plain English explanation CSS_PROP_COMMENTS = { 'margin': 'removes default spacing around elements', 'margin-top': 'spacing above the element', 'margin-bottom': 'spacing below the element', 'margin-left': 'spacing to the left of the element', 'margin-right': 'spacing to the right of the element', 'padding': 'inner spacing inside the element', 'padding-top': 'inner spacing above the content', 'padding-bottom': 'inner spacing below the content', 'padding-left': 'inner spacing on the left side', 'padding-right': 'inner spacing on the right side', 'display': 'controls how the element is rendered in the layout', 'flex-direction': 'sets the direction of flex items (row or column)', 'justify-content': 'aligns flex items along the main axis', 'align-items': 'aligns flex items along the cross axis', 'flex-wrap': 'allows flex items to wrap to the next line', 'gap': 'spacing between grid or flex children', 'position': 'controls how the element is positioned in the document', 'top': 'distance from the top of the positioned parent', 'bottom': 'distance from the bottom of the positioned parent', 'left': 'distance from the left of the positioned parent', 'right': 'distance from the right of the positioned parent', 'z-index': 'controls stacking order — higher values appear on top', 'width': 'horizontal size of the element', 'height': 'vertical size of the element', 'max-width': 'maximum width — prevents growing beyond this size', 'min-width': 'minimum width — prevents shrinking below this size', 'max-height': 'maximum height — prevents growing beyond this size', 'min-height': 'minimum height — prevents shrinking below this size', 'background': 'sets the background color or image', 'background-color': 'fills the element background with a solid color', 'background-image': 'sets an image as the background', 'color': 'sets the text color', 'font-size': 'controls the size of the text', 'font-weight': 'controls how bold or thin the text appears', 'font-family': 'sets the typeface (font) used for this element', 'line-height': 'spacing between lines of text', 'text-align': 'aligns text horizontally inside the element', 'text-decoration': 'adds underline, strikethrough, or other text decorations', 'letter-spacing': 'horizontal spacing between individual characters', 'border': 'draws a visible border around the element', 'border-radius': 'rounds the corners of the element', 'outline': 'similar to border but does not affect layout', 'box-shadow': 'adds a shadow effect behind the element', 'opacity': 'controls element transparency (0=invisible, 1=fully visible)', 'overflow': 'controls what happens when content overflows the element', 'cursor': 'changes the look of the mouse cursor on hover', 'transition': 'creates smooth animated changes when CSS properties change', 'transform': 'applies 2D/3D transformation (rotate, scale, translate)', 'animation': 'runs a CSS keyframe animation on this element', 'grid-template-columns': 'defines the column structure of the grid', 'grid-template-rows': 'defines the row structure of the grid', 'object-fit': 'controls how content (image/video) fits inside the element', 'pointer-events': 'controls whether the element responds to click events', 'white-space': 'controls how text whitespace is handled', 'content': 'inserts content before/after the element (used with ::before/::after)', } if comment_level in ["detailed", "educational"]: lines = css.split('\n') for line in lines: stripped = line.strip() if not stripped: output += line + "\n" continue # Selectors — annotate class/id/universal selectors if "{" in stripped and not stripped.startswith("@media"): m = re.match(r'([\w\s.,#:\-\[\]="\'()*>~+^$]+)\{', stripped) if m: selectors = m.group(1).strip().split(',') for sel in selectors: sel = sel.strip() if sel == '*': output += "/* Universal selector: applies to EVERY element on the page */\n" elif sel.startswith('.'): output += f"/* Class: {sel} — Styling for {sel} component */\n" elif sel.startswith('#'): output += f"/* ID: {sel} — Styling for the #{sel.lstrip('#')} element */\n" elif sel in ('body', 'html', 'header', 'footer', 'nav', 'main', 'section', 'article', 'aside'): output += f"/* Element: <{sel}> — base styles for the {sel} element */\n" output += line + "\n" continue # Media queries if stripped.startswith("@media"): bp_match = re.search(r'\((.*?)\)', stripped) bp = bp_match.group(1) if bp_match else "specific screen size" if "{" in line: parts = line.split("{", 1) output += parts[0] + "{\n /* Responsive: styles below only apply when " + bp + " */\n" + (parts[1] if parts[1].strip() else "") + "\n" else: output += line + "\n" output += f" /* Responsive: styles below only apply when {bp} */\n" continue # CSS property inline comments prop_match = re.match(r'\s*([\w-]+)\s*:', stripped) if prop_match and not stripped.startswith('/*') and not stripped.startswith('//'): prop = prop_match.group(1).lower() comment = CSS_PROP_COMMENTS.get(prop) if comment: indent = len(line) - len(line.lstrip()) output += ' ' * indent + f"/* {prop}: {comment} */\n" output += line + "\n" else: output += css style_tag.string = output def _annotate_js(soup, comment_level): """ Rule-based deterministic JS comments. """ for script in soup.find_all("script"): if script.get("src"): continue js = script.string or "" if not js.strip(): continue header = "// ═══ JAVASCRIPT — Interactive Logic ═══\n\n" output = header lines = js.split("\n") for line in lines: if comment_level in ["detailed", "educational"]: stripped = line.strip() # Fetch API calls fetch_match = re.search(r"fetch\(['\"]([^'\"]+)['\"]", stripped) if fetch_match: url = fetch_match.group(1) output += f"// API call: sends a network request to {url}\n" # Variable DOM Bindings var_match = re.search(r"(?:const|let|var)\s+([a-zA-Z0-9_]+)\s*=\s*document\.(getElementById|querySelector)\(['\"]([^'\"]+)['\"]\)", stripped) if var_match: var_name = var_match.group(1) sel_type = var_match.group(2) sel_val = var_match.group(3) output += f"// Variable: stores a reference to the {sel_val} element for later use\n" # DOM queries independently (if not caught by variable matcher above) elif "document.getElementById(" in stripped: m = re.search(r"document\.getElementById\(['\"]([^'\"]+)['\"]\)", stripped) if m: output += f"// DOM query: selects the element with id=\"{m.group(1)}\" from the page\n" elif "document.querySelectorAll(" in stripped: m = re.search(r"document\.querySelectorAll\(['\"]([^'\"]+)['\"]\)", stripped) if m: output += f"// DOM query: selects all elements matching \"{m.group(1)}\" as a list\n" elif "document.querySelector(" in stripped: m = re.search(r"document\.querySelector\(['\"]([^'\"]+)['\"]\)", stripped) if m: output += f"// DOM query: selects the first element matching \"{m.group(1)}\"\n" # Event Listeners event_match = re.search(r"addEventListener\(['\"]([^'\"]+)['\"]", stripped) if event_match: event = event_match.group(1) output += f"// Event listener: Handles {event} event\n" elif "onclick=" in stripped.replace(" ", "") or ".onclick" in stripped: output += f"// Event listener: Handles click event\n" # Function declarations func_match = re.match(r"function\s+([a-zA-Z0-9_]+)\s*\(", stripped) if func_match: name = func_match.group(1) output += f"// Function: {name} — handles {name} logic\n" else: const_func_match = re.match(r"(?:const|let|var)\s+([a-zA-Z0-9_]+)\s*=\s*(?:function|async\s+function|\([^)]*\)\s*=>)", stripped) if const_func_match: name = const_func_match.group(1) output += f"// Function: {name} — handles {name} logic\n" output += line + "\n" script.string = output def add_comments_to_webcraft_file( file_path=None, html_content=None, comment_level="concise", force_title=None, codet5_pipeline=None, dry_run=False, output_path=None ): """ Agent 1: Inject semantic code comments. Can accept either a file path or raw HTML content. Returns: The annotated HTML string, and writes to output_path if not dry_run. """ if not html_content and file_path: for enc in ["utf-8", "latin-1", "cp1252", "iso-8859-1"]: try: with open(file_path, "r", encoding=enc) as f: html_content = f.read() break except Exception: continue if not html_content: with open(file_path, "rb") as f: html_content = f.read().decode("utf-8", errors="ignore") soup = BeautifulSoup(html_content, "html.parser") # 1. Annotate HTML Tags using CodeT5 _annotate_html(soup, codet5_pipeline, comment_level) # 2. Annotate CSS _annotate_css(soup, comment_level) # 3. Annotate JS _annotate_js(soup, comment_level) if force_title: title_tag = soup.find("title") if title_tag: title_tag.string = force_title else: head = soup.find("head") if head: new_title = soup.new_tag("title") new_title.string = force_title head.insert(0, new_title) annotated_html = str(soup) if not dry_run: out_path = output_path or file_path if out_path: with open(out_path, "w", encoding="utf-8") as f: f.write(annotated_html) return annotated_html