Spaces:
Paused
Paused
| annotation_task_name: RAG Pipeline Evaluation | |
| data_files: | |
| - data/rag-traces.json | |
| item_properties: | |
| id_key: id | |
| text_key: task_description | |
| task_dir: . | |
| output_annotation_dir: annotation_output | |
| output_annotation_format: json | |
| instance_display: | |
| layout: | |
| direction: vertical | |
| gap: 16px | |
| fields: | |
| - key: task_description | |
| type: text | |
| label: User Query | |
| - key: metadata_table | |
| type: spreadsheet | |
| label: Pipeline Info | |
| display_options: | |
| compact: true | |
| max_height: 100 | |
| - key: conversation | |
| type: dialogue | |
| label: RAG Trace (Query -> Retrieval -> Synthesis) | |
| span_target: true | |
| display_options: | |
| show_turn_numbers: true | |
| alternating_shading: true | |
| annotation_schemes: | |
| - annotation_type: multirate | |
| name: retrieval_relevance | |
| description: Rate the relevance of each retrieved document to the query | |
| options: | |
| - name: doc_1 | |
| label: Document 1 | |
| - name: doc_2 | |
| label: Document 2 | |
| - name: doc_3 | |
| label: Document 3 | |
| - name: doc_4 | |
| label: Document 4 (if present) | |
| labels: | |
| - Irrelevant | |
| - Somewhat | |
| - Relevant | |
| - Highly Relevant | |
| - annotation_type: likert | |
| name: faithfulness | |
| description: Is the synthesized answer faithful to the retrieved documents? (No | |
| unsupported claims) | |
| min_label: Many unsupported claims | |
| max_label: Fully faithful | |
| size: 5 | |
| - annotation_type: likert | |
| name: completeness | |
| description: Does the answer cover all important aspects from the retrieved documents? | |
| min_label: Very incomplete | |
| max_label: Comprehensive | |
| size: 5 | |
| - annotation_type: radio | |
| name: citation_quality | |
| description: How well does the answer attribute claims to source documents? | |
| labels: | |
| - name: excellent | |
| tooltip: All claims properly attributed to correct sources | |
| - name: good | |
| tooltip: Most claims attributed, minor issues | |
| - name: fair | |
| tooltip: Some attributions missing or incorrect | |
| - name: poor | |
| tooltip: Few or no attributions, or many incorrect ones | |
| sequential_key_binding: true | |
| - annotation_type: span | |
| name: citation_spans | |
| description: Highlight any issues in the synthesized answer | |
| labels: | |
| - name: unsupported_claim | |
| tooltip: Claim not supported by any retrieved document | |
| - name: wrong_attribution | |
| tooltip: Claim attributed to wrong document | |
| - name: hallucination | |
| tooltip: Factually incorrect or fabricated information | |
| - name: missing_nuance | |
| tooltip: Source information oversimplified or distorted | |
| - annotation_type: likert | |
| name: answer_quality | |
| description: Overall quality of the synthesized answer | |
| min_label: Very poor | |
| max_label: Excellent | |
| size: 5 | |
| - annotation_type: radio | |
| name: misleading | |
| description: Could this answer mislead the user? | |
| labels: | |
| - name: not_misleading | |
| tooltip: Answer is accurate and balanced | |
| - name: slightly_misleading | |
| tooltip: Minor inaccuracies or imbalances | |
| - name: misleading | |
| tooltip: Answer could lead to wrong conclusions | |
| sequential_key_binding: true | |
| - annotation_type: text | |
| name: notes | |
| description: Additional observations about the RAG pipeline trace | |
| user_config: | |
| allow_all_users: true | |
| require_no_password: true | |