Add chat evaluation interface components and functionality

app.py

@@ -4,7 +4,6 @@ import json
 import datetime
 from pathlib import Path
 from huggingface_hub import InferenceClient, HfApi
-#from huggingface_hub import InferenceClient
 from config.constants import DEFAULT_SYSTEM_MESSAGE
 from config.settings import (
     HF_TOKEN, 
@@ -23,6 +22,15 @@ from web.training_interface import (
     register_model_action,
     start_finetune_action
 )
+from web.evaluation_interface import (
+    get_evaluation_status,
+    get_qa_pairs_dataframe,
+    load_qa_pair_for_evaluation,
+    save_evaluation,
+    generate_evaluation_report_html,
+    export_training_data_action
+)
+from src.analytics.chat_evaluator import ChatEvaluator
 
 if not HF_TOKEN:
     raise ValueError("HUGGINGFACE_TOKEN not found in environment variables")
@@ -81,6 +89,11 @@ ERROR_LOGS_PATH = os.path.join(os.path.dirname(__file__), "error_logs")
 client = None
 context_store = {}
 fallback_model_attempted = False
+chat_evaluator = ChatEvaluator(
+    hf_token=HF_TOKEN,
+    dataset_id=DATASET_ID,
+    chat_history_path=CHAT_HISTORY_PATH
+)
 
 print(f"Chat histories will be saved to: {CHAT_HISTORY_PATH}")
 
@@ -625,6 +638,73 @@ def save_parameters(model_key, max_len, temp, top_p_val, rep_pen):
     except Exception as e:
         return f"Error saving parameters: {str(e)}"
 
+def finetune_from_annotations(epochs=3, batch_size=4, learning_rate=2e-4, min_rating=4):
+    """
+    Fine-tune model using annotated QA pairs
+    
+    Args:
+        epochs: Number of training epochs
+        batch_size: Batch size for training
+        learning_rate: Learning rate
+        min_rating: Minimum average rating for including examples
+        
+    Returns:
+        (success, message)
+    """
+    try:
+        import tempfile
+        import os
+        from src.analytics.chat_evaluator import ChatEvaluator
+        from config.settings import HF_TOKEN, DATASET_ID, CHAT_HISTORY_PATH
+        
+        # Create evaluator
+        evaluator = ChatEvaluator(
+            hf_token=HF_TOKEN,
+            dataset_id=DATASET_ID,
+            chat_history_path=CHAT_HISTORY_PATH
+        )
+        
+        # Create temporary file for training data
+        with tempfile.NamedTemporaryFile(mode='w+', suffix='.jsonl', delete=False) as temp_file:
+            temp_path = temp_file.name
+        
+        # Export high-quality examples
+        success, message = evaluator.export_training_data(temp_path, min_rating)
+        
+        if not success:
+            return False, f"Failed to export training data: {message}"
+        
+        # Count examples
+        with open(temp_path, 'r') as f:
+            example_count = sum(1 for _ in f)
+        
+        if example_count == 0:
+            return False, "No high-quality examples found for fine-tuning"
+        
+        # Run actual fine-tuning using the export file
+        from src.training.fine_tuner import finetune_from_file
+        
+        success, message = finetune_from_file(
+            training_file=temp_path,
+            epochs=epochs,
+            batch_size=batch_size,
+            learning_rate=learning_rate
+        )
+        
+        # Clean up temporary file
+        try:
+            os.unlink(temp_path)
+        except:
+            pass
+        
+        if success:
+            return True, f"Successfully fine-tuned model with {example_count} annotated examples: {message}"
+        else:
+            return False, f"Fine-tuning failed: {message}"
+        
+    except Exception as e:
+        return False, f"Error during fine-tuning from annotations: {str(e)}"
+
 def initialize_app():
     """Initialize app with user preferences"""
     global client, ACTIVE_MODEL
@@ -810,14 +890,27 @@ with gr.Blocks() as demo:
             gr.Markdown("### Model Training Interface")
             
             with gr.Row():
-                with gr.Column():
-                    epochs = gr.Slider(minimum=1, maximum=10, value=3, step=1, label="Number of Epochs")
-                    batch_size = gr.Slider(minimum=1, maximum=32, value=4, step=1, label="Batch Size")
-                    learning_rate = gr.Slider(minimum=1e-6, maximum=1e-3, value=2e-4, label="Learning Rate")
+                with gr.Column(scale=1):
+                    training_tabs = gr.Tabs()
+                    
+                    with training_tabs:
+                        with gr.TabItem("Regular Training"):
+                            epochs = gr.Slider(minimum=1, maximum=10, value=3, step=1, label="Number of Epochs")
+                            batch_size = gr.Slider(minimum=1, maximum=32, value=4, step=1, label="Batch Size")
+                            learning_rate = gr.Slider(minimum=1e-6, maximum=1e-3, value=2e-4, label="Learning Rate")
+                            
+                            train_btn = gr.Button("Start Training", variant="primary")
+                            training_output = gr.Textbox(label="Training Status", interactive=False)
+                        
+                        with gr.TabItem("Train from Annotations"):
+                            annot_epochs = gr.Slider(minimum=1, maximum=10, value=3, step=1, label="Number of Epochs")
+                            annot_batch_size = gr.Slider(minimum=1, maximum=32, value=4, step=1, label="Batch Size")
+                            annot_learning_rate = gr.Slider(minimum=1e-6, maximum=1e-3, value=2e-4, label="Learning Rate")
+                            annot_min_rating = gr.Slider(minimum=1, maximum=5, value=4, step=0.5, label="Minimum Rating for Training")
+                            
+                            annot_train_btn = gr.Button("Start Training from Annotations", variant="primary")
+                            annot_training_output = gr.Textbox(label="Training Status", interactive=False)
                     
-                    train_btn = gr.Button("Start Training", variant="primary")
-                    training_output = gr.Textbox(label="Training Status", interactive=False)
-                     
                     gr.Markdown("""
                     <small>
                                         
@@ -834,10 +927,8 @@ with gr.Blocks() as demo:
                     2e-4 (0.0002) = Usually works best -> 1e-4 = Safer choice for fine-tuning
                     </small>
                     """)
-                    
-
 
-                with gr.Column():
+                with gr.Column(scale=1):
                     analysis_btn = gr.Button("Generate Chat Analysis")
                     analysis_output = gr.Markdown()
             
@@ -846,11 +937,140 @@ with gr.Blocks() as demo:
                 inputs=[epochs, batch_size, learning_rate],
                 outputs=[training_output]
             )
+            
+            # Function to handle training from annotations
+            def start_annotation_finetune(epochs, batch_size, learning_rate, min_rating):
+                """Wrapper function to start fine-tuning from annotations"""
+                success, message = finetune_from_annotations(
+                    epochs=epochs,
+                    batch_size=batch_size,
+                    learning_rate=learning_rate,
+                    min_rating=min_rating
+                )
+                return message
+            
+            annot_train_btn.click(
+                start_annotation_finetune,
+                inputs=[annot_epochs, annot_batch_size, annot_learning_rate, annot_min_rating],
+                outputs=[annot_training_output]
+            )
+            
             analysis_btn.click(
                 generate_chat_analysis,
                 inputs=[],
                 outputs=[analysis_output]
             )
+
+        with gr.Tab("Chat Evaluation"):
+            gr.Markdown("### Evaluation of Chat Responses")
+            
+            with gr.Row():
+                with gr.Column(scale=1):
+                    evaluation_status = gr.Markdown(get_evaluation_status(chat_evaluator))
+                    refresh_status_btn = gr.Button("Refresh Status")
+                    
+                    gr.Markdown("### Evaluation Metrics")
+                    evaluation_report = gr.HTML(generate_evaluation_report_html(chat_evaluator))
+                    refresh_report_btn = gr.Button("Refresh Report")
+                    
+                    gr.Markdown("### Export for Training")
+                    with gr.Row():
+                        min_rating = gr.Slider(
+                            minimum=1, 
+                            maximum=5, 
+                            value=4, 
+                            step=0.5, 
+                            label="Minimum Average Rating"
+                        )
+                        export_path = gr.Textbox(
+                            label="Export File Path (optional)",
+                            placeholder="Leave empty for default path"
+                        )
+                    export_btn = gr.Button("Export Annotated Data", variant="primary")
+                    export_status = gr.Textbox(label="Export Status", interactive=False)
+                    
+                with gr.Column(scale=2):
+                    show_evaluated = gr.Checkbox(label="Show Already Evaluated Pairs", value=False)
+                    qa_table = gr.DataFrame(get_qa_pairs_dataframe(chat_evaluator))
+                    
+                    gr.Markdown("### Select Conversation to Evaluate")
+                    selected_conversation = gr.Textbox(label="Conversation ID", placeholder="Select from table above")
+                    load_btn = gr.Button("Load Conversation", variant="primary")
+                    
+                    gr.Markdown("### Evaluate Response")
+                    question_display = gr.Textbox(label="User Question", interactive=False)
+                    original_answer = gr.TextArea(label="Original Bot Answer", interactive=False)
+                    improved_answer = gr.TextArea(label="Improved Answer (Gold Standard)", interactive=True)
+                    
+                    gr.Markdown("### Quality Ratings (1-5)")
+                    with gr.Row():
+                        accuracy = gr.Slider(minimum=1, maximum=5, value=3, step=1, label="Factual Accuracy")
+                        completeness = gr.Slider(minimum=1, maximum=5, value=3, step=1, label="Completeness")
+                    with gr.Row():
+                        relevance = gr.Slider(minimum=1, maximum=5, value=3, step=1, label="Relevance")
+                        clarity = gr.Slider(minimum=1, maximum=5, value=3, step=1, label="Clarity")
+                    legal_correctness = gr.Slider(minimum=1, maximum=5, value=3, step=1, label="Legal Correctness")
+                    
+                    notes = gr.TextArea(label="Evaluator Notes", placeholder="Add your notes about this response...")
+                    save_btn = gr.Button("Save Evaluation", variant="primary")
+                    evaluation_status_msg = gr.Textbox(label="Status", interactive=False)
+            
+            # Add event handlers
+            refresh_status_btn.click(
+                fn=get_evaluation_status,
+                inputs=[],
+                outputs=[evaluation_status],
+                kwargs={"evaluator": chat_evaluator}
+            )
+            
+            refresh_report_btn.click(
+                fn=generate_evaluation_report_html,
+                inputs=[],
+                outputs=[evaluation_report],
+                kwargs={"evaluator": chat_evaluator}
+            )
+            
+            show_evaluated.change(
+                fn=get_qa_pairs_dataframe,
+                inputs=[show_evaluated],
+                outputs=[qa_table],
+                kwargs={"evaluator": chat_evaluator}
+            )
+            
+            # Table selection to conversation ID textbox
+            qa_table.select(
+                fn=lambda df, evt: evt.value[0] if evt and evt.value and len(evt.value) > 0 else "",
+                inputs=[qa_table],
+                outputs=[selected_conversation]
+            )
+            
+            # Load conversation for evaluation
+            load_btn.click(
+                fn=load_qa_pair_for_evaluation,
+                inputs=[selected_conversation],
+                outputs=[question_display, original_answer, improved_answer, 
+                        accuracy, completeness, relevance, clarity, legal_correctness, notes],
+                kwargs={"evaluator": chat_evaluator}
+            )
+            
+            # Save evaluation
+            save_btn.click(
+                fn=save_evaluation,
+                inputs=[
+                    selected_conversation, question_display, original_answer, improved_answer,
+                    accuracy, completeness, relevance, clarity, legal_correctness, notes
+                ],
+                outputs=[evaluation_status_msg],
+                kwargs={"evaluator": chat_evaluator}
+            )
+            
+            # Export training data
+            export_btn.click(
+                fn=export_training_data_action,
+                inputs=[min_rating, export_path],
+                outputs=[export_status],
+                kwargs={"evaluator": chat_evaluator}
+            )
     
     # Model change handler
     model_selector.change(
@@ -882,4 +1102,4 @@ if __name__ == "__main__":
     if not load_vector_store():
         print("Knowledge base not found. Please create it through the interface.")
     
-    demo.launch()
+    demo.launch()

src/analytics/chat_evaluator.py

@@ -0,0 +1,320 @@
+"""
+Module for evaluation and annotation of bot responses
+"""
+
+import json
+import os
+import datetime
+from typing import List, Dict, Any, Tuple, Optional
+import pandas as pd
+from src.knowledge_base.dataset import DatasetManager
+from huggingface_hub import HfApi
+
+class ChatEvaluator:
+    def __init__(self, 
+                 dataset_manager: Optional[DatasetManager] = None, 
+                 hf_token: str = None,
+                 dataset_id: str = None,
+                 chat_history_path: str = None):
+        """
+        Initialize chat evaluator
+        
+        Args:
+            dataset_manager: Dataset manager for retrieving chat history
+            hf_token: Hugging Face token for uploading annotations
+            dataset_id: Hugging Face dataset ID
+            chat_history_path: Path to local chat history directory
+        """
+        self.dataset_manager = dataset_manager or DatasetManager()
+        self.hf_token = hf_token
+        self.dataset_id = dataset_id
+        self.chat_history_path = chat_history_path
+        self.annotations_dir = os.path.join(os.path.dirname(chat_history_path), "annotations") if chat_history_path else None
+        
+        # Create annotations directory if it doesn't exist
+        if self.annotations_dir:
+            os.makedirs(self.annotations_dir, exist_ok=True)
+    
+    def get_chat_history(self) -> List[Dict[str, Any]]:
+        """
+        Get all chat history data from local files and dataset
+        
+        Returns:
+            List of chat histories
+        """
+        success, chat_data = self.dataset_manager.get_chat_history()
+        if not success or not chat_data:
+            return []
+        return chat_data
+    
+    def get_qa_pairs_for_evaluation(self, limit: int = 50) -> List[Dict[str, Any]]:
+        """
+        Extract question-answer pairs for evaluation
+        
+        Args:
+            limit: Maximum number of pairs to return
+            
+        Returns:
+            List of QA pairs with metadata
+        """
+        chat_data = self.get_chat_history()
+        qa_pairs = []
+        
+        for chat in chat_data:
+            conversation_id = chat.get("conversation_id", "unknown")
+            timestamp = chat.get("timestamp", "")
+            history = chat.get("history", [])
+            
+            # Find user-assistant pairs in history
+            for i in range(len(history) - 1):
+                if history[i].get("role") == "user" and history[i+1].get("role") == "assistant":
+                    question = history[i].get("content", "").strip()
+                    answer = history[i+1].get("content", "").strip()
+                    
+                    # Only include non-empty pairs
+                    if question and answer:
+                        qa_pairs.append({
+                            "conversation_id": conversation_id,
+                            "timestamp": timestamp,
+                            "question": question,
+                            "original_answer": answer,
+                            "question_timestamp": history[i].get("timestamp", ""),
+                            "answer_timestamp": history[i+1].get("timestamp", "")
+                        })
+                        
+                        # Check if we've reached the limit
+                        if len(qa_pairs) >= limit:
+                            return qa_pairs
+        
+        return qa_pairs
+    
+    def get_evaluation_status(self) -> Dict[str, int]:
+        """
+        Get status of evaluated QA pairs
+        
+        Returns:
+            Dictionary with counts of evaluated and unevaluated QA pairs
+        """
+        all_pairs = self.get_qa_pairs_for_evaluation(limit=1000)  # Get a large sample
+        evaluated_pairs = self.get_annotations()
+        
+        # Count evaluated conversation IDs
+        evaluated_ids = set(item.get("conversation_id") for item in evaluated_pairs)
+        
+        return {
+            "total_qa_pairs": len(all_pairs),
+            "evaluated_pairs": len(evaluated_pairs),
+            "unevaluated_pairs": len(all_pairs) - len(evaluated_pairs),
+            "evaluated_conversations": len(evaluated_ids)
+        }
+    
+    def save_annotation(self, 
+                       conversation_id: str,
+                       question: str,
+                       original_answer: str,
+                       improved_answer: str,
+                       ratings: Dict[str, int],
+                       notes: str = "") -> Tuple[bool, str]:
+        """
+        Save evaluation annotation
+        
+        Args:
+            conversation_id: ID of the conversation
+            question: User question
+            original_answer: Original bot answer
+            improved_answer: Improved answer (gold standard)
+            ratings: Dictionary with ratings for different criteria
+            notes: Optional evaluator notes
+            
+        Returns:
+            (success, message)
+        """
+        if not self.annotations_dir:
+            return False, "Annotations directory not configured"
+        
+        try:
+            # Create annotation object
+            annotation = {
+                "conversation_id": conversation_id,
+                "timestamp": datetime.datetime.now().isoformat(),
+                "question": question,
+                "original_answer": original_answer,
+                "improved_answer": improved_answer,
+                "ratings": ratings,
+                "notes": notes
+            }
+            
+            # Create filename with conversation_id
+            filename = f"annotation_{conversation_id}.json"
+            filepath = os.path.join(self.annotations_dir, filename)
+            
+            # Save to local file
+            with open(filepath, 'w', encoding='utf-8') as f:
+                json.dump(annotation, f, ensure_ascii=False, indent=2)
+            
+            # Upload to HuggingFace dataset if configured
+            if self.hf_token and self.dataset_id:
+                try:
+                    api = HfApi(token=self.hf_token)
+                    
+                    # Extract just the directory name from annotations_dir
+                    dir_name = os.path.basename(self.annotations_dir)
+                    target_path = f"{dir_name}/{filename}"
+                    
+                    # Upload the file to the dataset
+                    api.upload_file(
+                        path_or_fileobj=filepath,
+                        path_in_repo=target_path,
+                        repo_id=self.dataset_id,
+                        repo_type="dataset"
+                    )
+                    
+                except Exception as e:
+                    return True, f"Saved locally but failed to upload to dataset: {str(e)}"
+            
+            return True, "Annotation saved successfully"
+        except Exception as e:
+            return False, f"Error saving annotation: {str(e)}"
+    
+    def get_annotations(self) -> List[Dict[str, Any]]:
+        """
+        Get all saved annotations
+        
+        Returns:
+            List of annotation objects
+        """
+        if not self.annotations_dir or not os.path.exists(self.annotations_dir):
+            return []
+        
+        annotations = []
+        for filename in os.listdir(self.annotations_dir):
+            if filename.startswith("annotation_") and filename.endswith(".json"):
+                try:
+                    filepath = os.path.join(self.annotations_dir, filename)
+                    with open(filepath, 'r', encoding='utf-8') as f:
+                        annotation = json.load(f)
+                        annotations.append(annotation)
+                except Exception as e:
+                    print(f"Error loading annotation {filename}: {str(e)}")
+        
+        # Sort by timestamp (newest first)
+        annotations.sort(key=lambda x: x.get("timestamp", ""), reverse=True)
+        return annotations
+    
+    def get_annotation_by_conversation_id(self, conversation_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Get annotation for a specific conversation
+        
+        Args:
+            conversation_id: Conversation ID to look for
+            
+        Returns:
+            Annotation object or None if not found
+        """
+        if not self.annotations_dir:
+            return None
+        
+        filepath = os.path.join(self.annotations_dir, f"annotation_{conversation_id}.json")
+        if os.path.exists(filepath):
+            try:
+                with open(filepath, 'r', encoding='utf-8') as f:
+                    return json.load(f)
+            except Exception as e:
+                print(f"Error loading annotation for {conversation_id}: {str(e)}")
+        
+        return None
+    
+    def export_training_data(self, output_file: str, min_rating: int = 4) -> Tuple[bool, str]:
+        """
+        Export high-quality annotated data for fine-tuning
+        
+        Args:
+            output_file: Path to output file
+            min_rating: Minimum average rating to include in training data
+            
+        Returns:
+            (success, message)
+        """
+        annotations = self.get_annotations()
+        
+        if not annotations:
+            return False, "No annotations available for export"
+        
+        try:
+            # Filter annotations by quality
+            high_quality_examples = []
+            
+            for annotation in annotations:
+                ratings = annotation.get("ratings", {})
+                
+                # Calculate average rating
+                if ratings:
+                    avg_rating = sum(ratings.values()) / len(ratings)
+                    
+                    # Include only high-quality examples
+                    if avg_rating >= min_rating:
+                        high_quality_examples.append({
+                            "messages": [
+                                {"role": "user", "content": annotation.get("question", "")},
+                                {"role": "assistant", "content": annotation.get("improved_answer", "")}
+                            ]
+                        })
+            
+            if not high_quality_examples:
+                return False, f"No examples meet the minimum quality threshold of {min_rating}"
+            
+            # Save to JSONL format
+            with open(output_file, "w", encoding="utf-8") as f:
+                for example in high_quality_examples:
+                    f.write(json.dumps(example, ensure_ascii=False) + "\n")
+            
+            return True, f"Successfully exported {len(high_quality_examples)} high-quality examples for training"
+        except Exception as e:
+            return False, f"Error exporting training data: {str(e)}"
+    
+    def generate_evaluation_report(self) -> Dict[str, Any]:
+        """
+        Generate evaluation summary report
+        
+        Returns:
+            Dictionary with evaluation metrics
+        """
+        annotations = self.get_annotations()
+        
+        if not annotations:
+            return {
+                "total_evaluations": 0,
+                "message": "No evaluations available"
+            }
+        
+        # Initialize metrics
+        criteria = set()
+        for annotation in annotations:
+            criteria.update(annotation.get("ratings", {}).keys())
+        
+        metrics = {
+            "total_evaluations": len(annotations),
+            "criteria_averages": {},
+            "overall_average": 0,
+            "improvement_rate": 0  # Percentage of answers that were improved
+        }
+        
+        # Calculate averages for each criterion
+        for criterion in criteria:
+            values = [a.get("ratings", {}).get(criterion, 0) for a in annotations if criterion in a.get("ratings", {})]
+            if values:
+                metrics["criteria_averages"][criterion] = sum(values) / len(values)
+        
+        # Calculate overall average
+        all_ratings = []
+        for annotation in annotations:
+            all_ratings.extend(annotation.get("ratings", {}).values())
+        
+        if all_ratings:
+            metrics["overall_average"] = sum(all_ratings) / len(all_ratings)
+        
+        # Calculate improvement rate
+        improved_count = sum(1 for a in annotations if a.get("original_answer") != a.get("improved_answer"))
+        metrics["improvement_rate"] = (improved_count / len(annotations)) * 100
+        
+        return metrics

src/training/fine_tuner.py

@@ -1,5 +1,5 @@
 """
-Модуль для дообучения языковой модели на основе собранных данных
+Module for fine-tuning a language model on collected data
 """
 
 import os
@@ -84,38 +84,38 @@ class FineTuner:
         try:
             logger.info(f"Загрузка модели {self.base_model_id}...")
             
-            # Загрузка токенизатора с использованием slow tokenizer
+            # Load tokenizer using slow tokenizer
             self.tokenizer = AutoTokenizer.from_pretrained(
                 self.base_model_id,
                 trust_remote_code=True,
-                use_fast=False  # Используем slow tokenizer
+                use_fast=False  # Using slow tokenizer
             )
             
-            # Специальные токены для диалогов
+            # Special tokens for dialogues
             special_tokens = {
                 "pad_token": "<PAD>",
                 "eos_token": "</s>",
                 "bos_token": "<s>",
-                "unk_token": "<unk>"  # Добавляем unknown token
+                "unk_token": "<unk>"  # Adding unknown token
             }
             
-            # Добавляем специальные токены, если их нет
+            # Add special tokens if they don't exist
             self.tokenizer.add_special_tokens({"additional_special_tokens": list(special_tokens.values())})
             
-            # Загрузка модели
+            # Load model
             self.model = AutoModelForCausalLM.from_pretrained(
                 self.base_model_id,
                 trust_remote_code=True,
                 device_map="auto" if self.device == "cuda" else None,
-                torch_dtype="auto"  # Автоматически выбираем оптимальный тип данных
+                torch_dtype="auto"  # Automatically choose optimal data type
             )
             
-            # Изменяем размер эмбеддингов для новых токенов
+            # Resize embeddings for new tokens
             self.model.resize_token_embeddings(len(self.tokenizer))
             
-            logger.info("Модель и токенизатор успешно загружены")
+            logger.info("Model and tokenizer loaded successfully")
         except Exception as e:
-            logger.error(f"Ошибка при загрузке модели: {str(e)}")
+            logger.error(f"Error loading model: {str(e)}")
             raise
     
     def setup_lora_config(
@@ -125,17 +125,17 @@ class FineTuner:
         lora_dropout: float = 0.05
     ) -> LoraConfig:
         """
-        Настройка конфигурации LoRA для эффективного дообучения
+        Setup LoRA configuration for efficient fine-tuning
         
         Args:
-            r: Ранг матриц LoRA
-            lora_alpha: Альфа параметр LoRA
-            lora_dropout: Вероятность dropout в LoRA слоях
+            r: Rank of LoRA matrices
+            lora_alpha: LoRA alpha parameter
+            lora_dropout: Dropout probability in LoRA layers
             
         Returns:
-            Конфигурация LoRA
+            LoRA configuration
         """
-        # Создаем конфигурацию LoRA
+        # Create LoRA configuration
         lora_config = LoraConfig(
             task_type=TaskType.CAUSAL_LM,
             r=r,
@@ -149,34 +149,34 @@ class FineTuner:
     
     def prepare_model_for_training(self):
         """
-        Подготовка модели к обучению с использованием LoRA
+        Prepare model for training using LoRA
         """
         if self.model is None:
             self.load_model_and_tokenizer()
         
-        # Настройка LoRA
+        # Setup LoRA
         lora_config = self.setup_lora_config()
         
-        # Применяем LoRA к модели
+        # Apply LoRA to model
         self.model = get_peft_model(self.model, lora_config)
         
-        # Вывод информации о параметрах
+        # Output parameter information
         trainable_params = sum(p.numel() for p in self.model.parameters() if p.requires_grad)
         all_params = sum(p.numel() for p in self.model.parameters())
-        logger.info(f"Обучаемых параметров: {trainable_params:,} из {all_params:,} ({trainable_params/all_params:.2%})")
+        logger.info(f"Trainable parameters: {trainable_params:,} of {all_params:,} ({trainable_params/all_params:.2%})")
     
     def tokenize_dataset(self, dataset):
         """
-        Токенизация датасета для обучения
+        Tokenize dataset for training
         
         Args:
-            dataset: Датасет для токенизации
+            dataset: Dataset to tokenize
             
         Returns:
-            То��енизированный датасет
+            Tokenized dataset
         """
         def tokenize_function(examples):
-            # Форматируем диалоги в единую строку
+            # Format dialogues into single string
             texts = []
             for dialog in examples["messages"]:
                 text = ""
@@ -187,7 +187,7 @@ class FineTuner:
                         text += f"Assistant: {message['content']}\n"
                 texts.append(text)
             
-            # Токенизируем тексты
+            # Tokenize texts
             tokenized = self.tokenizer(
                 texts,
                 padding="max_length",
@@ -198,7 +198,7 @@ class FineTuner:
             
             return tokenized
         
-        # Применяем функцию токенизации
+        # Apply tokenization function
         tokenized_dataset = dataset.map(
             tokenize_function,
             batched=True,
@@ -207,6 +207,75 @@ class FineTuner:
         
         return tokenized_dataset
     
+    # Добавить этот метод в класс fine_tuner.py или в функции модуля:
+
+def finetune_from_annotations(epochs=3, batch_size=4, learning_rate=2e-4, min_rating=4):
+    """
+    Fine-tune model using annotated QA pairs
+    
+    Args:
+        epochs: Number of training epochs
+        batch_size: Batch size for training
+        learning_rate: Learning rate
+        min_rating: Minimum average rating for including examples
+        
+    Returns:
+        (success, message)
+    """
+    try:
+        import tempfile
+        import os
+        from src.analytics.chat_evaluator import ChatEvaluator
+        from config.settings import HF_TOKEN, DATASET_ID, CHAT_HISTORY_PATH
+        
+        # Create evaluator
+        evaluator = ChatEvaluator(
+            hf_token=HF_TOKEN,
+            dataset_id=DATASET_ID,
+            chat_history_path=CHAT_HISTORY_PATH
+        )
+        
+        # Create temporary file for training data
+        with tempfile.NamedTemporaryFile(mode='w+', suffix='.jsonl', delete=False) as temp_file:
+            temp_path = temp_file.name
+        
+        # Export high-quality examples
+        success, message = evaluator.export_training_data(temp_path, min_rating)
+        
+        if not success:
+            return False, f"Failed to export training data: {message}"
+        
+        # Count examples
+        with open(temp_path, 'r') as f:
+            example_count = sum(1 for _ in f)
+        
+        if example_count == 0:
+            return False, "No high-quality examples found for fine-tuning"
+        
+        # Run actual fine-tuning using the export file
+        from src.training.fine_tuner import finetune_from_file
+        
+        success, message = finetune_from_file(
+            training_file=temp_path,
+            epochs=epochs,
+            batch_size=batch_size,
+            learning_rate=learning_rate
+        )
+        
+        # Clean up temporary file
+        try:
+            os.unlink(temp_path)
+        except:
+            pass
+        
+        if success:
+            return True, f"Successfully fine-tuned model with {example_count} annotated examples: {message}"
+        else:
+            return False, f"Fine-tuning failed: {message}"
+        
+    except Exception as e:
+        return False, f"Error during fine-tuning from annotations: {str(e)}"
+    
     def train(
         self,
         training_data_path: Optional[str] = None,
@@ -218,49 +287,49 @@ class FineTuner:
         save_strategy: str = "epoch"
     ) -> Tuple[bool, str]:
         """
-        Запуск процесса дообучения модели
+        Start model fine-tuning process
         
         Args:
-            training_data_path: Путь к данным для обучения (если None, данные будут подготовлены автоматически)
-            num_train_epochs: Количество эпох обучения
-            per_device_train_batch_size: Размер батча на устройство
-            gradient_accumulation_steps: Количество шагов накопления градиента
-            learning_rate: Скорость обучения
-            logging_steps: Частота логирования
-            save_strategy: Стратегия сохранения модели
+            training_data_path: Path to training data (if None, data will be prepared automatically)
+            num_train_epochs: Number of training epochs
+            per_device_train_batch_size: Batch size per device
+            gradient_accumulation_steps: Number of gradient accumulation steps
+            learning_rate: Learning rate
+            logging_steps: Logging frequency
+            save_strategy: Model saving strategy
             
         Returns:
-            (успех, сообщение)
+            (success, message)
         """
         try:
-            # Подгот��вка данных для обучения, если не указан путь
+            # Prepare training data if path not specified
             if training_data_path is None:
                 training_data_path = self.prepare_training_data()
                 temp_data = True
             else:
                 temp_data = False
             
-            # Загрузка модели и токенизатора, если не загружены
+            # Load model and tokenizer if not loaded
             if self.model is None or self.tokenizer is None:
                 self.load_model_and_tokenizer()
             
-            # Подготовка модели для обучения
+            # Prepare model for training
             self.prepare_model_for_training()
             
-            # Загрузка датасета
+            # Load dataset
             dataset = load_dataset("json", data_files=training_data_path, split="train")
-            logger.info(f"Загружено {len(dataset)} примеров из {training_data_path}")
+            logger.info(f"Loaded {len(dataset)} examples from {training_data_path}")
             
-            # Токенизация датасета
+            # Tokenize dataset
             tokenized_dataset = self.tokenize_dataset(dataset)
             
-            # Создание колатора данных
+            # Create data collator
             data_collator = DataCollatorForLanguageModeling(
                 tokenizer=self.tokenizer,
                 mlm=False
             )
             
-            # Настройка аргументов обучения
+            # Setup training arguments
             training_args = TrainingArguments(
                 output_dir=self.output_dir,
                 num_train_epochs=num_train_epochs,
@@ -278,7 +347,7 @@ class FineTuner:
                 load_best_model_at_end=True
             )
             
-            # Создание тренера
+            # Create trainer
             trainer = Trainer(
                 model=self.model,
                 args=training_args,
@@ -287,23 +356,23 @@ class FineTuner:
                 tokenizer=self.tokenizer
             )
             
-            # Запуск обучения
-            logger.info("Начало обучения модели...")
+            # Start training
+            logger.info("Starting model training...")
             trainer.train()
             
-            # Сохранение модели
-            logger.info(f"Сохранение обученной модели в {self.output_dir}")
+            # Save model
+            logger.info(f"Saving trained model to {self.output_dir}")
             trainer.save_model(self.output_dir)
             self.tokenizer.save_pretrained(self.output_dir)
             
-            # Удаляем временный файл, если он был создан
+            # Remove temporary file if created
             if temp_data and os.path.exists(training_data_path):
                 os.remove(training_data_path)
             
-            return True, f"Модель успешно обучена и сохранена в {self.output_dir}"
+            return True, f"Model successfully trained and saved to {self.output_dir}"
         except Exception as e:
-            logger.error(f"Ошибка в процессе обучения: {str(e)}")
-            return False, f"Ошибка в процессе обучения: {str(e)}"
+            logger.error(f"Error during training: {str(e)}")
+            return False, f"Error during training: {str(e)}"
     
     def upload_model_to_hub(
         self, 
@@ -312,24 +381,24 @@ class FineTuner:
         token: Optional[str] = None
     ) -> Tuple[bool, str]:
         """
-        Загрузка обученной модели на Hugging Face Hub
+        Upload trained model to Hugging Face Hub
         
         Args:
-            repo_id: Идентификатор репозитория на Hugging Face Hub
-            private: Флаг приватности репозитория
-            token: Токен доступа к Hugging Face Hub
+            repo_id: Repository ID on Hugging Face Hub
+            private: Repository privacy flag
+            token: Hugging Face Hub access token
             
         Returns:
-            (успех, сообщение)
+            (success, message)
         """
         try:
             if not os.path.exists(os.path.join(self.output_dir, "pytorch_model.bin")):
-                return False, "Обученная модель не найдена. Сначала выполните обучение."
+                return False, "Trained model not found. Please train the model first."
             
-            # Инициализация API
+            # Initialize API
             api = HfApi(token=token)
             
-            # Загрузка модели на Hub
+            # Upload model to Hub
             api.create_repo(repo_id=repo_id, private=private, repo_type="model", exist_ok=True)
             api.upload_folder(
                 folder_path=self.output_dir,
@@ -337,35 +406,35 @@ class FineTuner:
                 repo_type="model"
             )
             
-            return True, f"Модель успешно загружена на Hugging Face Hub: {repo_id}"
+            return True, f"Model successfully uploaded to Hugging Face Hub: {repo_id}"
         except Exception as e:
-            return False, f"Ошибка при загрузке модели на Hub: {str(e)}"
+            return False, f"Error uploading model to Hub: {str(e)}"
 
 def finetune_from_chat_history(epochs: int = 3) -> Tuple[bool, str]:
     """
-    Функция для запуска процесса дообучения на основе истории чатов
+    Function to start fine-tuning process based on chat history
     
     Args:
-        epochs: Количество эпох обучения
+        epochs: Number of training epochs
         
     Returns:
-        (успех, сообщение)
+        (success, message)
     """
-    # Анализ чатов и подготовка данных
+    # Analyze chats and prepare data
     analyzer = ChatAnalyzer()
     report = analyzer.generate_analytics_report()
     
-    # Проверка наличия достаточного количества данных
+    # Check if there's enough data
     if report["qa_pairs_count"] < 10:
-        return False, f"Недостаточно данных для дообучения. Найдено всего {report['qa_pairs_count']} пар вопрос-ответ."
+        return False, f"Insufficient data for fine-tuning. Only {report['qa_pairs_count']} QA pairs found."
     
-    # Создание и запуск процесса дообучения
+    # Create and start fine-tuning process
     tuner = FineTuner()
     success, message = tuner.train(num_train_epochs=epochs)
     
     return success, message
 
 if __name__ == "__main__":
-    # Пример использования
+    # Usage example
     success, message = finetune_from_chat_history()
     print(message)

web/evaluation_interface.py

@@ -0,0 +1,240 @@
+"""
+Interface components for chat evaluation
+"""
+
+import gradio as gr
+import pandas as pd
+from src.analytics.chat_evaluator import ChatEvaluator
+import json
+import os
+from typing import Dict, Any, List, Tuple
+
+def get_evaluation_status(evaluator: ChatEvaluator) -> str:
+    """
+    Format evaluation status for display
+    
+    Args:
+        evaluator: ChatEvaluator instance
+        
+    Returns:
+        Formatted markdown string with status information
+    """
+    status = evaluator.get_evaluation_status()
+    
+    status_md = f"""
+    ## Evaluation Status
+    
+    - **Total QA Pairs:** {status['total_qa_pairs']}
+    - **Evaluated Pairs:** {status['evaluated_pairs']} ({status['evaluated_pairs']/max(1, status['total_qa_pairs'])*100:.1f}%)
+    - **Unevaluated Pairs:** {status['unevaluated_pairs']}
+    - **Evaluated Conversations:** {status['evaluated_conversations']}
+    """
+    
+    return status_md
+
+def get_qa_pairs_dataframe(evaluator: ChatEvaluator, show_evaluated: bool = False, limit: int = 50) -> pd.DataFrame:
+    """
+    Get QA pairs as a pandas DataFrame for display
+    
+    Args:
+        evaluator: ChatEvaluator instance
+        show_evaluated: Whether to show already evaluated pairs
+        limit: Maximum number of pairs to return
+        
+    Returns:
+        DataFrame with QA pairs
+    """
+    qa_pairs = evaluator.get_qa_pairs_for_evaluation(limit=200)  # Get more than needed for filtering
+    annotations = evaluator.get_annotations()
+    
+    # Create set of evaluated conversation IDs
+    evaluated_ids = set(a.get("conversation_id") for a in annotations)
+    
+    # Filter QA pairs based on show_evaluated parameter
+    if not show_evaluated:
+        qa_pairs = [pair for pair in qa_pairs if pair.get("conversation_id") not in evaluated_ids]
+    
+    # Limit the results
+    qa_pairs = qa_pairs[:limit]
+    
+    # Create DataFrame
+    if qa_pairs:
+        df = pd.DataFrame(qa_pairs)
+        
+        # Add "Evaluated" column
+        df["evaluated"] = df["conversation_id"].apply(lambda x: "Yes" if x in evaluated_ids else "No")
+        
+        # Select and rename columns for display
+        display_df = df[["conversation_id", "question", "original_answer", "evaluated"]].copy()
+        display_df = display_df.rename(columns={
+            "conversation_id": "ID",
+            "question": "Question",
+            "original_answer": "Answer",
+            "evaluated": "Evaluated"
+        })
+        
+        # Truncate long text for better display
+        display_df["Question"] = display_df["Question"].apply(lambda x: (x[:150] + "...") if len(x) > 150 else x)
+        display_df["Answer"] = display_df["Answer"].apply(lambda x: (x[:150] + "...") if len(x) > 150 else x)
+        
+        return display_df
+    
+    # Return empty DataFrame if no pairs
+    return pd.DataFrame(columns=["ID", "Question", "Answer", "Evaluated"])
+
+def load_qa_pair_for_evaluation(evaluator: ChatEvaluator, conversation_id: str) -> Tuple[str, str, Dict, str]:
+    """
+    Load a QA pair for evaluation
+    
+    Args:
+        evaluator: ChatEvaluator instance
+        conversation_id: ID of the conversation to load
+        
+    Returns:
+        Tuple of (question, original_answer, existing_ratings, notes)
+    """
+    # Get all QA pairs
+    qa_pairs = evaluator.get_qa_pairs_for_evaluation(limit=1000)
+    
+    # Find the requested pair
+    for pair in qa_pairs:
+        if pair.get("conversation_id") == conversation_id:
+            question = pair.get("question", "")
+            original_answer = pair.get("original_answer", "")
+            
+            # Check if there's an existing annotation
+            annotation = evaluator.get_annotation_by_conversation_id(conversation_id)
+            
+            if annotation:
+                existing_ratings = annotation.get("ratings", {})
+                improved_answer = annotation.get("improved_answer", original_answer)
+                notes = annotation.get("notes", "")
+                return question, original_answer, improved_answer, existing_ratings, notes
+            
+            return question, original_answer, original_answer, {}, ""
+    
+    return "", "", "", {}, ""
+
+def save_evaluation(
+    evaluator: ChatEvaluator,
+    conversation_id: str,
+    question: str,
+    original_answer: str,
+    improved_answer: str,
+    accuracy: int,
+    completeness: int,
+    relevance: int,
+    clarity: int,
+    legal_correctness: int,
+    notes: str
+) -> str:
+    """
+    Save evaluation to file and dataset
+    
+    Args:
+        evaluator: ChatEvaluator instance
+        conversation_id: ID of the conversation
+        question: User question
+        original_answer: Original bot answer
+        improved_answer: Improved answer
+        accuracy: Rating for factual accuracy (1-5)
+        completeness: Rating for completeness (1-5)
+        relevance: Rating for relevance (1-5)
+        clarity: Rating for clarity (1-5)
+        legal_correctness: Rating for legal correctness (1-5)
+        notes: Evaluator notes
+        
+    Returns:
+        Status message
+    """
+    # Create ratings dictionary
+    ratings = {
+        "accuracy": accuracy,
+        "completeness": completeness,
+        "relevance": relevance,
+        "clarity": clarity,
+        "legal_correctness": legal_correctness
+    }
+    
+    # Save annotation
+    success, message = evaluator.save_annotation(
+        conversation_id=conversation_id,
+        question=question,
+        original_answer=original_answer,
+        improved_answer=improved_answer,
+        ratings=ratings,
+        notes=notes
+    )
+    
+    return message
+
+def generate_evaluation_report_html(evaluator: ChatEvaluator) -> str:
+    """
+    Generate HTML report of evaluation metrics
+    
+    Args:
+        evaluator: ChatEvaluator instance
+        
+    Returns:
+        HTML string with report
+    """
+    report = evaluator.generate_evaluation_report()
+    
+    if report["total_evaluations"] == 0:
+        return "<p>No evaluations available yet.</p>"
+    
+    # Format criteria averages
+    criteria_html = ""
+    for criterion, avg in report["criteria_averages"].items():
+        # Calculate stars representation (1-5)
+        stars = "★" * int(avg) + "☆" * (5 - int(avg))
+        criteria_html += f"""
+        <tr>
+            <td>{criterion.capitalize()}</td>
+            <td>{avg:.2f}/5.0</td>
+            <td>{stars}</td>
+        </tr>
+        """
+    
+    # Overall stars representation
+    overall_stars = "★" * int(report["overall_average"]) + "☆" * (5 - int(report["overall_average"]))
+    
+    html = f"""
+    <div style="padding: 15px; border: 1px solid #ccc; border-radius: 5px; margin-top: 10px;">
+        <h3>Evaluation Report</h3>
+        
+        <p><strong>Total Evaluations:</strong> {report["total_evaluations"]}</p>
+        <p><strong>Overall Average Rating:</strong> {report["overall_average"]:.2f}/5.0 {overall_stars}</p>
+        <p><strong>Improvement Rate:</strong> {report["improvement_rate"]:.1f}% of responses were improved</p>
+        
+        <h4>Criteria Ratings:</h4>
+        <table style="width: 100%; border-collapse: collapse;">
+            <tr>
+                <th style="text-align: left; padding: 8px; border-bottom: 1px solid #ddd;">Criterion</th>
+                <th style="text-align: left; padding: 8px; border-bottom: 1px solid #ddd;">Average Score</th>
+                <th style="text-align: left; padding: 8px; border-bottom: 1px solid #ddd;">Rating</th>
+            </tr>
+            {criteria_html}
+        </table>
+    </div>
+    """
+    
+    return html
+
+def export_training_data_action(evaluator: ChatEvaluator, min_rating: int, output_file: str) -> str:
+    """
+    Action for exporting training data
+    
+    Args:
+        evaluator: ChatEvaluator instance
+        min_rating: Minimum average rating (1-5)
+        output_file: Output file path
+        
+    Returns:
+        Status message
+    """
+    if not output_file:
+        output_file = os.path.join(os.path.dirname(evaluator.annotations_dir), "training_data.jsonl")
+    
+    success, message = evaluator.export_training_data(output_file, min_rating)
+    return message