""" Multi-Criteria Rubric Evaluation Layout Rate items on multiple criteria simultaneously in a structured grid. THE missing schema for LLM evaluation — enables MT-Bench-style multi-dimensional scoring. Research: Zheng et al. (2023) "Judging LLM-as-a-judge with MT-Bench"; Ke et al. (2024) "CritiqueLLM". """ import logging from .identifier_utils import ( safe_generate_layout, generate_element_identifier, generate_validation_attribute, escape_html_content, generate_layout_attributes ) logger = logging.getLogger(__name__) DEFAULT_SCALE_POINTS = 5 DEFAULT_SCALE_LABELS = ["Poor", "Below Average", "Average", "Good", "Excellent"] def generate_rubric_eval_layout(annotation_scheme): """ Generate HTML for a Multi-Criteria Rubric Evaluation interface. Args: annotation_scheme (dict): Configuration including: - name: Schema identifier - description: Display description - scale_points: Number of scale points (default 5) - scale_labels: Labels for each scale point - criteria: List of {name, description} dicts - show_overall: Whether to include an "Overall" row Returns: tuple: (html_string, key_bindings) """ return safe_generate_layout(annotation_scheme, _generate_rubric_eval_layout_internal) def _generate_rubric_eval_layout_internal(annotation_scheme): schema_name = annotation_scheme['name'] description = annotation_scheme['description'] scale_points = annotation_scheme.get('scale_points', DEFAULT_SCALE_POINTS) scale_labels = annotation_scheme.get('scale_labels', DEFAULT_SCALE_LABELS[:scale_points]) criteria = annotation_scheme.get('criteria', []) show_overall = annotation_scheme.get('show_overall', False) if not criteria: raise ValueError(f"rubric_eval schema '{schema_name}' requires at least one criterion in 'criteria'") # Pad or trim scale_labels to match scale_points while len(scale_labels) < scale_points: scale_labels.append(f"{len(scale_labels) + 1}") scale_labels = scale_labels[:scale_points] layout_attrs = generate_layout_attributes(annotation_scheme) validation = generate_validation_attribute(annotation_scheme) # Build header row header_cells = 'Criterion' for i, label in enumerate(scale_labels): header_cells += f'{escape_html_content(label)}
{i + 1}' # Build criterion rows rows_html = "" all_criteria = list(criteria) if show_overall: all_criteria.append({"name": "overall", "description": "Overall quality"}) for criterion in all_criteria: crit_name = criterion['name'] crit_desc = criterion.get('description', '') crit_label = f"{schema_name}:{crit_name}" identifiers = generate_element_identifier(schema_name, crit_name, "radio") # Override name: each criterion row needs its own radio group # (generate_element_identifier uses schema-only name for radios, # but rubric_eval needs per-criterion groups) row_group_name = f"{escape_html_content(schema_name)}:::{escape_html_content(crit_name)}" cells = f"""
{escape_html_content(crit_name)}
{f'
{escape_html_content(crit_desc)}
' if crit_desc else ''} """ for i in range(scale_points): value = str(i + 1) radio_id = f"{identifiers['id']}-{value}" cells += f""" """ is_overall_class = " rubric-overall-row" if crit_name == "overall" else "" rows_html += f'{cells}' html = f"""
{escape_html_content(description)}
{header_cells} {rows_html}
""" logger.info(f"Generated rubric eval layout for {schema_name} with {len(criteria)} criteria") return html, []