app.py

app.py

@@ -1,4 +1,3 @@
-# app_refactored.py - 重构后的展示系统
 import gradio as gr
 import json
 import pandas as pd
@@ -10,6 +9,7 @@ from openai import OpenAI
 import re
 import spacy
 from spellchecker import SpellChecker
+import difflib
 
 # ======================== API配置 ========================
 DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY", "")
@@ -60,6 +60,109 @@ Next, please correct the following sentence according to the above requirements.
 
 [input]: """
 
+# ======================== 新增：颜色对比函数 ========================
+def generate_colored_diff(original, cleaned):
+    """
+    生成带颜色标注的HTML差异对比
+    原始文本中的错误：红色
+    清洗后的修正：绿色
+    """
+    # 分词处理
+    original_words = original.split()
+    cleaned_words = cleaned.split()
+    
+    # 使用difflib进行序列匹配
+    matcher = difflib.SequenceMatcher(None, original_words, cleaned_words)
+    
+    original_html = []
+    cleaned_html = []
+    
+    for tag, i1, i2, j1, j2 in matcher.get_opcodes():
+        if tag == 'equal':
+            # 相同部分保持黑色
+            original_html.extend(original_words[i1:i2])
+            cleaned_html.extend(cleaned_words[j1:j2])
+        elif tag == 'replace':
+            # 替换部分：原文红色，新文绿色
+            original_html.extend([f'<span style="color: #dc3545; font-weight: bold;">{w}</span>' 
+                                 for w in original_words[i1:i2]])
+            cleaned_html.extend([f'<span style="color: #28a745; font-weight: bold;">{w}</span>' 
+                                for w in cleaned_words[j1:j2]])
+        elif tag == 'delete':
+            # 删除部分：原文红色带删除线
+            original_html.extend([f'<span style="color: #dc3545; text-decoration: line-through;">{w}</span>' 
+                                 for w in original_words[i1:i2]])
+        elif tag == 'insert':
+            # 插入部分：新文绿色
+            cleaned_html.extend([f'<span style="color: #28a745; font-weight: bold;">{w}</span>' 
+                                for w in cleaned_words[j1:j2]])
+    
+    return ' '.join(original_html), ' '.join(cleaned_html)
+
+def create_comparison_html(original_list, cleaned_list):
+    """
+    创建HTML表格展示对比
+    """
+    html = """
+    <div style="font-family: 'Segoe UI', Arial, sans-serif; max-width: 100%; overflow-x: auto;">
+        <style>
+            .comparison-table {
+                width: 100%;
+                border-collapse: collapse;
+                margin: 20px 0;
+                box-shadow: 0 2px 8px rgba(0,0,0,0.1);
+            }
+            .comparison-table th {
+                background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+                color: white;
+                padding: 12px;
+                text-align: left;
+                font-weight: 600;
+            }
+            .comparison-table td {
+                padding: 12px;
+                border-bottom: 1px solid #e0e0e0;
+                line-height: 1.6;
+            }
+            .comparison-table tr:hover {
+                background-color: #f8f9fa;
+            }
+            .index-col {
+                width: 50px;
+                text-align: center;
+                font-weight: bold;
+                color: #6c757d;
+            }
+        </style>
+        <table class="comparison-table">
+            <thead>
+                <tr>
+                    <th class="index-col">#</th>
+                    <th>原始问题 (红色=错误)</th>
+                    <th>清洗后问题 (绿色=修正)</th>
+                </tr>
+            </thead>
+            <tbody>
+    """
+    
+    for idx, (orig, clean) in enumerate(zip(original_list, cleaned_list), 1):
+        orig_colored, clean_colored = generate_colored_diff(str(orig), str(clean))
+        html += f"""
+                <tr>
+                    <td class="index-col">{idx}</td>
+                    <td>{orig_colored}</td>
+                    <td>{clean_colored}</td>
+                </tr>
+        """
+    
+    html += """
+            </tbody>
+        </table>
+    </div>
+    """
+    
+    return html
+
 # ======================== 工具函数 ========================
 def check_api_key():
     if not DEEPSEEK_API_KEY:
@@ -164,52 +267,40 @@ def calculate_spelling_error_density(sentences):
 
 # ======================== Leaderboard数据处理 ========================
 def load_leaderboard_data():
-    """从JSON加载Leaderboard数据（现在包含Category字段）"""
     json_path = "leaderboard.json"
     try:
         with open(json_path, 'r', encoding='utf-8') as f:
             data = json.load(f)
-        
-        # Category已经在JSON中定义，直接加载即可
         return pd.DataFrame(data)
     except Exception as e:
         print(f"Error loading leaderboard: {e}")
         return pd.DataFrame()
 
-def make_clickable_download(download_text):
-    """将Markdown链接转换为HTML链接"""
-    if '[下载](' in download_text:
-        url = download_text.split('(')[1].rstrip(')')
-        return f'<a href="{url}" class="download-link" target="_blank">下载</a>'
-    return download_text
-
 def filter_leaderboard(df, query):
-    """根据Category筛选Leaderboard"""
     if query == "all":
         return df
     else:
         return df[df['Category'] == query]
 
 def search_leaderboard(df, query):
-    """搜索Leaderboard"""
     if not query:
         return df
     return df[df['Benchmark'].str.contains(query, case=False, na=False)]
 
-# ======================== 数据清洗函数 ========================
+# ======================== 数据清洗函数（修改版）========================
 def clean_dataset(file_path, question_column, model_choice, temperature, max_samples, progress=gr.Progress()):
     try:
         try:
             check_api_key()
         except ValueError as e:
-            return str(e), None, None
+            return str(e), None, None, ""
         
         progress(0.05, desc="📁 读取数据文件...")
         df = pd.read_parquet(file_path)
         
         if question_column not in df.columns:
             available_columns = ", ".join(df.columns.tolist())
-            return f"❌ 列名 '{question_column}' 不存在！\n可用列名: {available_columns}", None, None
+            return f"❌ 列名 '{question_column}' 不存在！\n可用列名: {available_columns}", None, None, ""
         
         data_ori = df[question_column].tolist()[:int(max_samples)]
         total = len(data_ori)
@@ -320,6 +411,9 @@ def clean_dataset(file_path, question_column, model_choice, temperature, max_sam
         log_text += f"   变化: {delta_sed:+.2f}% {'✅ 改善' if delta_sed < 0 else '⚠️ 增加'}\n"
         log_text += f"{'='*50}\n"
         
+        # 生成带颜色的对比HTML
+        preview_html = create_comparison_html(data_ori[:5], lst_final[:5])
+        
         preview_df = pd.DataFrame({
             '原始问题': [str(x)[:100] for x in data_ori[:5]],
             '清洗后问题': [str(x)[:100] for x in lst_final[:5]]
@@ -327,12 +421,12 @@ def clean_dataset(file_path, question_column, model_choice, temperature, max_sam
         
         progress(1.0, desc="✅ 完成！")
         
-        return log_text, output_path, preview_df
+        return log_text, output_path, preview_df, preview_html
         
     except Exception as e:
         import traceback
         error_detail = traceback.format_exc()
-        return f"❌ 处理出错: {str(e)}\n\n详细错误:\n{error_detail}", None, None
+        return f"❌ 处理出错: {str(e)}\n\n详细错误:\n{error_detail}", None, None, ""
 
 # ======================== 文本内容 ========================
 ABOUT_TEXT = """
@@ -369,11 +463,18 @@ ABOUT_TEXT = """
 - **CoQA**: 对话问答
 - 以及更多...
 
+### 颜色标注说明
+
+- 🔴 **红色**: 原始文本中的错误（拼写、语法、空格等）
+- 🟢 **绿色**: 清洗后的修正内容
+- ⚫ **黑色**: 未修改的正确部分
+
 ### 技术栈
 
 - **LLM**: DeepSeek API (deepseek-r1-distill-llama-8b)
 - **前端**: Gradio 4.16.0
 - **数据处理**: Pandas + PyArrow (Parquet)
+- **差异对比**: Python difflib
 - **API调用**: OpenAI SDK
 - **部署**: Hugging Face Spaces
 
@@ -382,63 +483,15 @@ ABOUT_TEXT = """
 - **WAR (Whitespace Anomaly Rate)**: 空白符异常率
 - **SED (Spelling Error Density)**: 拼写错误密度
 
-### 数据集分类
-
-- **BT (Basic Tasks)**: 基础任务 - MRPC, RTE, SST2
-- **RA (Reasoning Abilities)**: 推理能力 - ARC, GSM8K, MMLU
-- **TG (Text Generation)**: 文本生成 - CoQA, DROP, Truthful_QA
-- **SU (Speech Understanding)**: 语音理解 - WNLI, Natural_questions
-- **ME (Medical)**: 医学领域 - MedMCQA, MedQA, PubMedQA
-- **GR (Grammatical)**: 语法领域 - BEA-2019, CoNLL-2014
-
-### 使用说明
-
-1. **配置 API Key**: Settings → Repository secrets → `DEEPSEEK_API_KEY`
-2. **上传数据集**: 选择 `.parquet` 文件
-3. **指定列名**: 输入包含问题的列名（通常是 `question`）
-4. **调整参数**: 选择模型、temperature等
-5. **开始清洗**: 点击按钮开始处理
-6. **下载结果**: 下载 `XXX-Denoising.parquet` 文件
-
-⚠️ **重要提示**:
-- Demo版本限制最多处理100个样本
-- 完整版本可处理数万样本
-- 建议 temperature=0.1 以获得稳定输出
-
 ---
 
 **研究生毕业论文成果展示** | Powered by DeepSeek API
 """
 
-SUBMISSION_TEXT = """
-## 提交说明
-
-### 如何提交新的去噪结果
-
-1. **准备数据**: 使用本系统对benchmark数据集进行去噪
-2. **记录指标**: 记录WAR和SED指标
-3. **提交PR**: 在GitHub上提交Pull Request
-4. **审核**: 等待维护者审核
-
-### 数据格式要求
-
-提交的数据需要包含以下字段:
-- ID: 序号
-- Category: 类别 (BT/RA/TG/SU/ME/GR)
-- Benchmark名称
-- WAR (%)
-- SED
-- Download: 下载链接
-
-### 联系方式
-
-如有问题,请通过以下方式联系:
-- GitHub Issues
-- Email: your-email@example.com
-"""
-
 # ======================== Gradio界面 ========================
-demo = gr.Blocks(title="数据集清洗框架展示系统")
+demo = gr.Blocks(title="数据集清洗框架展示系统", css="""
+    .markdown-text { font-size: 16px; line-height: 1.6; }
+""")
 
 with demo:
     gr.Markdown(
@@ -449,11 +502,9 @@ with demo:
         elem_classes="markdown-text"
     )
     
-    # 加载leaderboard数据
     leaderboard_data = load_leaderboard_data()
     
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        # ==================== Tab 1: Evaluation Table ====================
         with gr.TabItem("📊 Evaluation Table", id=0):
             with gr.Column():
                 gr.Markdown("### 清洗效果排行榜")
@@ -484,7 +535,6 @@ with demo:
                     visible=False
                 )
                 
-                # 绑定搜索和筛选
                 search_bar.submit(
                     lambda df, query: search_leaderboard(df, query)[['ID', 'Category', 'Benchmark', 'WAR', 'SED', 'Download']],
                     [hidden_leaderboard, search_bar],
@@ -502,22 +552,15 @@ with demo:
                 - **Category**: BT=基础任务, RA=推理能力, TG=文本生成, SU=语音理解, ME=医学领域, GR=语法领域
                 - **WAR**: 空白符异常率变化 (正值表示改善)
                 - **SED**: 拼写错误密度变化 (负值表示改善)
-                - 绿色: 正向提升 | 红色: 负向影响
                 """, elem_classes="markdown-text")
         
-        # ==================== Tab 2: Performance Plot ====================
         with gr.TabItem("📈 Performance Plot", id=1):
             gr.Markdown("### 性能可视化分析")
             gr.Markdown("**注意**: 性能图表功能开发中,敬请期待。")
-            
-            # 这里可以添加性能图表
-            # 例如: WAR和SED的对比图、不同方法的效果对比等
         
-        # ==================== Tab 3: About ====================
         with gr.TabItem("📝 About", id=2):
             gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
         
-        # ==================== Tab 4: Submit Results ====================
         with gr.TabItem("🚀 Submit Results", id=3):
             gr.Markdown("## 提交去噪结果")
             
@@ -565,28 +608,29 @@ with demo:
                         max_lines=15
                     )
                     
-                    preview_df = gr.Dataframe(
-                        label="🔍 结果预览",
-                        wrap=True
-                    )
-                    
                     download_file = gr.File(label="📥 下载去噪后的数据集")
             
+            # 新增：颜色对比预览区域
+            gr.Markdown("### 🎨 清洗效果对比预览")
+            gr.Markdown("""
+            **颜色说明**: 
+            - 🔴 <span style="color: #dc3545;">红色</span> = 原始文本中的错误
+            - 🟢 <span style="color: #28a745;">绿色</span> = 清洗后的修正
+            - ⚫ 黑色 = 未修改的正确部分
+            """)
+            
+            colored_preview = gr.HTML(label="带颜色标注的对比")
+            
+            preview_df = gr.Dataframe(
+                label="🔍 原始对比表格",
+                wrap=True
+            )
+            
             clean_btn.click(
                 fn=clean_dataset,
                 inputs=[file_input, question_column, model_choice, temperature, max_samples],
-                outputs=[output_text, download_file, preview_df]
+                outputs=[output_text, download_file, preview_df, colored_preview]
             )
-            
-            gr.Markdown("""
-            ### WAC-GEC方法 (开发中)
-            
-            WAC-GEC (Whitespace Anomaly Correction - Grammar Error Correction) 方法结合了:
-            - 空白符异常检测与修正
-            - 语法错误检测与修正
-            
-            该功能即将上线,敬请期待!
-            """, elem_classes="markdown-text")
 
 if __name__ == "__main__":
     demo.launch(