Upload 6 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+banner.png filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 AG
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,16 @@
----
-license: mit
----
+---  
+license: mit  
+language:  
+- en  
+---
+
+<img src="banner.png" alt="AgGPT-19 Banner" width="100%">
+
+# AgGPT-19
+
+## Heavy Duty intelligence
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+# HEAVY
+AgGPT-19 is a powerful AI language model designed to assist with a wide range of tasks, from answering questions to generating creative content. It is built on advanced machine learning techniques and trained on diverse datasets to ensure high-quality responses.

chat.py

@@ -0,0 +1,522 @@
+import os
+import re
+import random
+from typing import List, Dict, Tuple, Any
+from collections import defaultdict
+import math
+from feather import FeatherManager, similarity_score
+from train import GrammarRules, PatternExtractor
+
+
+class SimpleReasoningEngine:
+    """Basic reasoning engine for AgGPT-19"""
+    
+    def __init__(self):
+        self.logical_patterns = {
+            'cause_effect': ['because', 'since', 'therefore', 'as a result', 'leads to'],
+            'comparison': ['better than', 'worse than', 'similar to', 'different from', 'compared to'],
+            'sequence': ['first', 'then', 'next', 'finally', 'after that'],
+            'conditional': ['if', 'unless', 'provided that', 'in case'],
+        }
+        
+        self.context_memory = {}
+    
+    def analyze_intent(self, user_input: str, context: List[Tuple[str, str]]) -> Dict[str, Any]:
+        """Analyze user intent and context"""
+        input_lower = user_input.lower()
+        
+        intent = {
+            'type': 'unknown',
+            'topic': '',
+            'emotion': 'neutral',
+            'requires_reasoning': False,
+            'context_dependent': False
+        }
+        
+        # Detect intent type
+        if any(word in input_lower for word in ['what', 'how', 'why', 'when', 'where', 'who']):
+            intent['type'] = 'question'
+        elif any(word in input_lower for word in ['hello', 'hi', 'hey', 'good morning']):
+            intent['type'] = 'greeting'
+        elif any(word in input_lower for word in ['please', 'can you', 'help me']):
+            intent['type'] = 'request'
+        elif any(word in input_lower for word in ['thank', 'thanks', 'bye', 'goodbye']):
+            intent['type'] = 'social'
+        
+        # Detect if reasoning is needed
+        if any(pattern in input_lower for patterns in self.logical_patterns.values() for pattern in patterns):
+            intent['requires_reasoning'] = True
+        
+        # Check context dependency
+        if any(word in input_lower for word in ['this', 'that', 'it', 'they', 'what about']):
+            intent['context_dependent'] = True
+        
+        # Extract topic
+        words = [word for word in input_lower.split() if len(word) > 3]
+        if words:
+            intent['topic'] = ' '.join(words[:3])
+        
+        return intent
+    
+    def apply_reasoning(self, intent: Dict[str, Any], knowledge: Dict[str, str], context: List[Tuple[str, str]]) -> str:
+        """Apply basic reasoning to generate more intelligent responses"""
+        
+        if intent['context_dependent'] and context:
+            # Use recent context
+            recent_context = ' '.join([item[0] + ' ' + item[1] for item in context[-2:]])
+            return f"Building on our conversation: {recent_context[:100]}..."
+        
+        if intent['requires_reasoning']:
+            # Apply logical reasoning patterns
+            if intent['type'] == 'question' and 'why' in intent['topic']:
+                return "This is likely due to several factors that work together..."
+            elif 'how' in intent['topic']:
+                return "Here's a step-by-step approach to understand this..."
+        
+        return ""
+
+
+class ResponseGenerator: 
+    def __init__(self, feather_manager: FeatherManager):
+        self.feather_manager = feather_manager
+        self.pattern_extractor = PatternExtractor()
+        self.grammar_rules = GrammarRules()
+        self.models = []
+        self.context_window = []
+        self.max_context_length = 10
+        self.knowledge_cache = {}
+        
+        # Enhanced response generation for AgGPT-19
+        self.reasoning_engine = SimpleReasoningEngine()
+        self.response_templates = {
+            'greeting': ["Hello! {topic}", "Hi there! {topic}", "Good {time_of_day}! {topic}"],
+            'question': ["Based on what I know, {answer}", "Let me think about that. {answer}", "{answer}"],
+            'explanation': ["Here's how it works: {explanation}", "The key point is {explanation}", "{explanation}"],
+            'request': ["I'd be happy to help with {request}", "Sure, I can assist with {request}", "Let me help you with {request}"]
+        }
+        
+    def load_models(self):
+        print("Loading enhanced mini-models...")
+        self.models = self.feather_manager.load_all_models()
+        print(f"Loaded {len(self.models)} enhanced mini-models")
+        
+        if not self.models:
+            print("No trained models found! Please run train.py first.")
+            return False
+        
+        # Build knowledge cache from all models
+        self._build_knowledge_cache()
+        
+        return True
+    
+    def _build_knowledge_cache(self):
+        """Build a unified knowledge cache from all models"""
+        for model in self.models:
+            knowledge_base = model.get('knowledge_base', {})
+            if knowledge_base:
+                self.knowledge_cache.update(knowledge_base)
+        
+        print(f"Built knowledge cache with {len(self.knowledge_cache)} entries")
+    
+    def calculate_model_scores(self, user_input: str) -> List[Tuple[Dict[str, Any], float]]:
+        if not self.models:
+            return []
+        
+        input_pattern = self.pattern_extractor.create_pattern(user_input)
+        input_keywords = set(self.pattern_extractor.extract_keywords(user_input))
+        
+        model_scores = []
+        
+        for model in self.models:
+            score = 0.0
+            pattern_matches = 0
+            keyword_matches = 0
+            
+            for pattern in model.get('patterns', []):
+                pattern_sim = self.pattern_extractor.calculate_pattern_similarity(input_pattern, pattern)
+                score += pattern_sim
+                if pattern_sim > 0.3:
+                    pattern_matches += 1
+            
+            model_keywords = set(model.get('keywords', []))
+            if model_keywords and input_keywords:
+                keyword_overlap = len(input_keywords.intersection(model_keywords))
+                keyword_total = len(input_keywords.union(model_keywords))
+                keyword_score = keyword_overlap / keyword_total if keyword_total > 0 else 0
+                score += keyword_score * 2
+                keyword_matches = keyword_overlap
+            
+            confidence = model.get('confidence', 0.5)
+            score *= confidence
+            
+            training_samples = model.get('training_samples', 1)
+            training_bonus = min(0.2, training_samples / 100)
+            score += training_bonus
+            
+            context_bonus = self._calculate_context_bonus(user_input, model)
+            score += context_bonus
+            
+            model_scores.append((model, score))
+        
+        model_scores.sort(key=lambda x: x[1], reverse=True)
+        
+        return model_scores
+    
+    def _calculate_context_bonus(self, user_input: str, model: Dict[str, Any]) -> float:
+        if not self.context_window:
+            return 0.0
+        
+        context_bonus = 0.0
+        
+        for prev_input, prev_response in self.context_window[-3:]:
+            for pattern in model.get('patterns', [])[:5]:
+                pattern_sim = similarity_score(prev_input, pattern.strip())
+                context_bonus += pattern_sim * 0.1
+        
+        return min(context_bonus, 0.3)
+    
+    def select_top_models(self, model_scores: List[Tuple[Dict[str, Any], float]], top_k: int = 5) -> List[Tuple[Dict[str, Any], float]]:
+        valid_models = [(model, score) for model, score in model_scores if score > 0.01]
+        
+        if not valid_models:
+            valid_models = random.sample(model_scores, min(3, len(model_scores)))
+        
+        return valid_models[:top_k]
+    
+    def generate_responses_from_models(self, user_input: str, top_models: List[Tuple[Dict[str, Any], float]]) -> List[Tuple[str, float]]:
+        responses = []
+        input_pattern = self.pattern_extractor.create_pattern(user_input)
+        
+        for model, model_score in top_models:
+            model_responses = []
+            best_similarity = 0.0
+            
+            patterns = model.get('patterns', [])
+            model_responses_list = model.get('responses', [])
+            
+            if not patterns or not model_responses_list:
+                continue
+            
+            best_matches = []
+            for i, pattern in enumerate(patterns):
+                if i < len(model_responses_list):
+                    sim = self.pattern_extractor.calculate_pattern_similarity(input_pattern, pattern)
+                    if sim > 0.1:
+                        best_matches.append((model_responses_list[i], sim))
+            
+            best_matches.sort(key=lambda x: x[1], reverse=True)
+            
+            selected_responses = best_matches[:3] if best_matches else [(random.choice(model_responses_list), 0.1)]
+            
+            for response, pattern_sim in selected_responses:
+                weight = model_score * (0.7 + pattern_sim * 0.3)
+                responses.append((response, weight))
+        
+        return responses
+    
+    def combine_responses(self, responses: List[Tuple[str, float]], intent: Dict[str, Any] = None, reasoning_response: str = "") -> str:
+        """Intelligently combine responses using context and reasoning"""
+        if not responses:
+            return "I'm not sure how to respond to that."
+        
+        # If we have reasoning response, prioritize it
+        if reasoning_response:
+            best_response = responses[0][0] if responses else ""
+            if best_response:
+                combined = f"{reasoning_response} {best_response}"
+                return combined[:300]  # Limit length
+            return reasoning_response
+        
+        filtered_responses = [(resp, weight) for resp, weight in responses if weight > 0.05]
+        if not filtered_responses:
+            filtered_responses = responses[:1]
+        
+        # Smart response grouping
+        response_groups = defaultdict(list)
+        for response, weight in filtered_responses:
+            # Group by semantic similarity rather than just first words
+            key = self._get_semantic_key(response)
+            response_groups[key].append((response, weight))
+        
+        best_responses = []
+        for group in response_groups.values():
+            best_resp, best_weight = max(group, key=lambda x: x[1])
+            best_responses.append((best_resp, best_weight))
+        
+        # Select best response intelligently
+        if len(best_responses) > 1:
+            # Consider intent when selecting response
+            if intent and intent.get('type') == 'greeting':
+                # Prefer shorter, friendlier responses for greetings
+                best_responses.sort(key=lambda x: len(x[0]) + (0 if any(word in x[0].lower() for word in ['hello', 'hi', 'good']) else 100))
+            elif intent and intent.get('type') == 'question':
+                # Prefer longer, more informative responses for questions
+                best_responses.sort(key=lambda x: -len(x[0]))
+            
+            selected_response = best_responses[0][0]
+        else:
+            selected_response = best_responses[0][0]
+        
+        # Apply response template if appropriate
+        if intent:
+            templated_response = self._apply_response_template(selected_response, intent)
+            if templated_response:
+                selected_response = templated_response
+        
+        final_response = selected_response
+        
+        if not final_response.endswith('<eos>'):
+            final_response += ' <eos>'
+        
+        return final_response
+    
+    def _get_semantic_key(self, response: str) -> str:
+        """Get semantic grouping key for response"""
+        words = response.lower().split()
+        
+        # Group by intent/sentiment
+        if any(word in words for word in ['hello', 'hi', 'good']):
+            return 'greeting'
+        elif any(word in words for word in ['thank', 'welcome']):
+            return 'gratitude'
+        elif any(word in words for word in ['sorry', 'apologize']):
+            return 'apology'
+        elif len(words) > 20:
+            return 'explanation'
+        else:
+            return 'simple'
+    
+    def _apply_response_template(self, response: str, intent: Dict[str, Any]) -> str:
+        """Apply response templates for more natural responses"""
+        intent_type = intent.get('type', 'unknown')
+        
+        if intent_type in self.response_templates:
+            templates = self.response_templates[intent_type]
+            template = random.choice(templates)
+            
+            # Simple template filling
+            if '{topic}' in template:
+                template = template.replace('{topic}', intent.get('topic', ''))
+            if '{answer}' in template:
+                template = template.replace('{answer}', response[:100])
+            if '{explanation}' in template:
+                template = template.replace('{explanation}', response[:150])
+            if '{request}' in template:
+                template = template.replace('{request}', intent.get('topic', 'that'))
+            
+            return template
+        
+        return ""
+    
+    def generate_response(self, user_input: str) -> str:
+        if not user_input.strip():
+            return "Please say something! <eos>"
+        
+        # Enhanced response generation for AgGPT-19
+        # 1. Analyze intent and context
+        intent = self.reasoning_engine.analyze_intent(user_input, self.context_window)
+        
+        # 2. Check for direct knowledge match
+        knowledge_response = self._check_knowledge_base(user_input, intent)
+        if knowledge_response:
+            self.context_window.append((user_input, knowledge_response))
+            if len(self.context_window) > self.max_context_length:
+                self.context_window.pop(0)
+            return knowledge_response
+        
+        # 3. Apply reasoning if needed
+        reasoning_response = self.reasoning_engine.apply_reasoning(intent, self.knowledge_cache, self.context_window)
+        
+        # 4. Get model-based responses
+        model_scores = self.calculate_model_scores(user_input)
+        
+        if not model_scores:
+            fallback_response = self._generate_fallback_response(intent)
+            self.context_window.append((user_input, fallback_response))
+            return fallback_response
+        
+        top_models = self.select_top_models(model_scores, top_k=5)
+        responses = self.generate_responses_from_models(user_input, top_models)
+        
+        # 5. Intelligently combine responses
+        final_response = self.combine_responses(responses, intent, reasoning_response)
+        
+        # 6. Apply post-processing
+        final_response = self._post_process_response(final_response, intent)
+        
+        self.context_window.append((user_input, final_response))
+        if len(self.context_window) > self.max_context_length:
+            self.context_window.pop(0)
+        
+        return final_response
+    
+    def _check_knowledge_base(self, user_input: str, intent: Dict[str, Any]) -> str:
+        """Check if we have direct knowledge to answer this query"""
+        input_lower = user_input.lower()
+        
+        # Look for exact matches first
+        for key, value in self.knowledge_cache.items():
+            if key.replace('_', ' ') in input_lower or any(word in key for word in input_lower.split() if len(word) > 3):
+                return f"{value} <eos>"
+        
+        return ""
+    
+    def _generate_fallback_response(self, intent: Dict[str, Any]) -> str:
+        """Generate intelligent fallback responses"""
+        if intent['type'] == 'greeting':
+            return random.choice([
+                "Hello! How can I help you today?",
+                "Hi there! What would you like to know?",
+                "Good day! I'm here to assist you."
+            ]) + " <eos>"
+        elif intent['type'] == 'question':
+            return "That's an interesting question. I'd need to learn more about that topic to give you a good answer. <eos>"
+        elif intent['type'] == 'request':
+            return "I'd like to help you with that. Could you provide more details about what you need? <eos>"
+        else:
+            return "I understand what you're saying. Could you tell me more about that? <eos>"
+    
+    def _post_process_response(self, response: str, intent: Dict[str, Any]) -> str:
+        """Apply post-processing to make responses more natural"""
+        if not response.endswith('<eos>'):
+            response += ' <eos>'
+        
+        # Add conversational elements based on intent
+        if intent['type'] == 'question' and not any(phrase in response.lower() for phrase in ['based on', 'according to']):
+            response = response.replace('<eos>', '').strip()
+            response = f"From what I understand, {response} <eos>"
+        
+        # Ensure proper capitalization
+        response = response[0].upper() + response[1:] if response else response
+        
+        return response
+    
+    def get_model_statistics(self) -> Dict[str, Any]:
+        if not self.models:
+            return {"total_models": 0}
+        
+        total_patterns = sum(len(model.get('patterns', [])) for model in self.models)
+        total_responses = sum(len(model.get('responses', [])) for model in self.models)
+        avg_confidence = sum(model.get('confidence', 0) for model in self.models) / len(self.models)
+        total_training_samples = sum(model.get('training_samples', 0) for model in self.models)
+        
+        return {
+            "total_models": len(self.models),
+            "total_patterns": total_patterns,
+            "total_responses": total_responses,
+            "average_confidence": avg_confidence,
+            "total_training_samples": total_training_samples
+        }
+
+
+class AgGPTChat:
+    
+    def __init__(self, models_dir: str = "models"):
+        self.feather_manager = FeatherManager(models_dir)
+        self.response_generator = ResponseGenerator(self.feather_manager)
+        self.conversation_history = []
+    
+    def initialize(self) -> bool:
+        print("AgGPT-19 Enhanced Intelligence Chat System")
+        print("=" * 50)
+        
+        success = self.response_generator.load_models()
+        if success:
+            stats = self.response_generator.get_model_statistics()
+            print(f"Enhanced Model Statistics:")
+            print(f"   Mini-models loaded: {stats['total_models']}")
+            print(f"   Total patterns: {stats['total_patterns']}")
+            print(f"   Total responses: {stats['total_responses']}")
+            print(f"   Average confidence: {stats['average_confidence']:.3f}")
+            print(f"   Training samples: {stats['total_training_samples']}")
+            print(f"   Knowledge entries: {len(self.response_generator.knowledge_cache)}")
+            print("=" * 50)
+            print("Enhanced chat initialized! Type 'quit' to exit.")
+            print("Features: Context awareness, semantic understanding, reasoning!")
+            print()
+        
+        return success
+    
+    def chat_loop(self):
+        if not self.initialize():
+            return
+        
+        while True:
+            try:
+                user_input = input("You: ").strip()
+                
+                if not user_input:
+                    continue
+                
+                if user_input.lower() in ['quit', 'exit', 'bye', 'goodbye']:
+                    print("AgGPT-19: Goodbye! Thanks for chatting with me! <eos>")
+                    break
+                
+                if user_input.lower() in ['stats', 'statistics']:
+                    stats = self.response_generator.get_model_statistics()
+                    print("Current Statistics:")
+                    for key, value in stats.items():
+                        print(f"   {key}: {value}")
+                    continue
+                
+                if user_input.lower() in ['clear', 'reset']:
+                    self.response_generator.context_window = []
+                    print("Context cleared!")
+                    continue
+                
+                print("AgGPT: ", end="", flush=True)
+                response = self.response_generator.generate_response(user_input)
+                
+                display_response = response.replace(' <eos>', '').replace('<eos>', '')
+                print(display_response)
+                print()
+                
+                self.conversation_history.append({
+                    'user': user_input,
+                    'assistant': display_response
+                })
+                
+            except KeyboardInterrupt:
+                print("\n\nAgGPT-19: Chat interrupted. Goodbye!")
+                break
+            except Exception as e:
+                print(f"\nError: {e}")
+                print("Let me try again...")
+                continue
+    
+    def batch_test(self, test_inputs: List[str]):
+        if not self.initialize():
+            return
+        
+        print("Running enhanced batch test...")
+        print("=" * 50)
+        
+        for i, test_input in enumerate(test_inputs, 1):
+            print(f"Test {i}: {test_input}")
+            response = self.response_generator.generate_response(test_input)
+            display_response = response.replace(' <eos>', '').replace('<eos>', '')
+            print(f"Response: {display_response}")
+            print("-" * 30)
+
+
+def main():
+    chat = AgGPTChat()
+    
+    import sys
+    if len(sys.argv) > 1 and sys.argv[1] == "test":
+        test_inputs = [
+            "hi",
+            "hello there",
+            "how are you?",
+            "what's your favorite color?",
+            "tell me a joke",
+            "thank you",
+            "goodbye"
+        ]
+        chat.batch_test(test_inputs)
+    else:
+        chat.chat_loop()
+
+
+if __name__ == "__main__":
+    main()

feather.py

@@ -0,0 +1,424 @@
+import pandas as pd
+import os
+from typing import Dict, List, Any, Optional
+import math
+
+
+class FeatherManager:
+    
+    def __init__(self, models_dir: str = "models"):
+        self.models_dir = models_dir
+        os.makedirs(models_dir, exist_ok=True)
+    
+    def save_mini_model(self, model_data: Dict[str, Any], model_id: int) -> str:
+        filename = f"AgGPT_Expert_{model_id:04d}.feather"
+        filepath = os.path.join(self.models_dir, filename)
+        
+        patterns = model_data.get('patterns', [])
+        responses = model_data.get('responses', [])
+        
+        if not patterns or not responses:
+            print(f"Warning: Model {model_id} has empty patterns or responses")
+            patterns = patterns or ['hello']
+            responses = responses or ['Hello!']
+        
+        # FIXED: Efficient data structure for AgGPT-19
+        # Store only pattern-level data in main table
+        df_data = {
+            'patterns': [str(pattern) for pattern in patterns],
+            'responses': [str(response) for response in responses],
+            'weights': model_data.get('weights', [1.0] * len(patterns)),
+            'response_templates': model_data.get('response_templates', [{}] * len(patterns)),
+        }
+        
+        # Ensure all arrays are same length
+        max_len = len(patterns)
+        for key, value in df_data.items():
+            if isinstance(value, list):
+                while len(value) < max_len:
+                    if key == 'weights':
+                        value.append(1.0)
+                    elif key == 'response_templates':
+                        value.append({})
+                    else:
+                        value.append('')
+        
+        # Store model-level metadata separately (NO DUPLICATION!)
+        model_metadata = {
+            'model_id': model_id,
+            'confidence': model_data.get('confidence', 0.5),
+            'keywords': ' '.join(model_data.get('keywords', [])),
+            'training_samples': model_data.get('training_samples', 0),
+            'knowledge_base': str(model_data.get('knowledge_base', {})),
+            'semantic_categories': str(model_data.get('semantic_categories', {})),
+            'grammar_rules': str(model_data.get('grammar_rules', []))
+        }
+        
+        # Create main patterns DataFrame
+        df_patterns = pd.DataFrame(df_data)
+        
+        # Create single-row metadata DataFrame
+        df_metadata = pd.DataFrame([model_metadata])
+        
+        # Save both as separate sheets in the same feather file using a different approach
+        # Since feather doesn't support multiple sheets, we'll store metadata as the first row
+        # and add a special marker
+        
+        # Create combined structure with metadata as first row
+        metadata_row = {
+            'patterns': f"__METADATA__{model_metadata['model_id']}",
+            'responses': model_metadata['knowledge_base'],
+            'weights': float(model_metadata['confidence']),
+            'response_templates': f"{model_metadata['keywords']}|{model_metadata['training_samples']}|{model_metadata['semantic_categories']}|{model_metadata['grammar_rules']}"
+        }
+        
+        # Combine metadata row with pattern data
+        combined_data = [metadata_row]
+        
+        for i in range(len(patterns)):
+            row = {
+                'patterns': df_data['patterns'][i],
+                'responses': df_data['responses'][i], 
+                'weights': df_data['weights'][i],
+                'response_templates': str(df_data['response_templates'][i])
+            }
+            combined_data.append(row)
+        
+        df_combined = pd.DataFrame(combined_data)
+        df_combined.to_feather(filepath)
+        
+        print(f"Saved optimized mini-model: {filename} ({len(patterns)} patterns + metadata)")
+        return filepath
+    
+    def load_mini_model(self, model_id: int) -> Optional[Dict[str, Any]]:
+        filename = f"AgGPT_Expert_{model_id:04d}.feather"
+        filepath = os.path.join(self.models_dir, filename)
+        
+        if not os.path.exists(filepath):
+            return None
+        
+        try:
+            df = pd.read_feather(filepath)
+            
+            # FIXED: Load optimized structure for AgGPT-19
+            if len(df) == 0:
+                return None
+            
+            # First row contains metadata (marked with __METADATA__)
+            if df['patterns'].iloc[0].startswith('__METADATA__'):
+                # Extract metadata from first row
+                metadata_parts = df['response_templates'].iloc[0].split('|')
+                
+                model_data = {
+                    'model_id': model_id,
+                    'confidence': float(df['weights'].iloc[0]),
+                    'keywords': metadata_parts[0].split() if len(metadata_parts) > 0 else [],
+                    'training_samples': int(metadata_parts[1]) if len(metadata_parts) > 1 and metadata_parts[1].isdigit() else 0,
+                }
+                
+                # Parse knowledge base from responses column
+                try:
+                    kb_str = df['responses'].iloc[0]
+                    model_data['knowledge_base'] = eval(kb_str) if kb_str != '{}' else {}
+                except:
+                    model_data['knowledge_base'] = {}
+                
+                # Parse semantic categories
+                try:
+                    if len(metadata_parts) > 2:
+                        model_data['semantic_categories'] = eval(metadata_parts[2]) if metadata_parts[2] != '{}' else {}
+                    else:
+                        model_data['semantic_categories'] = {}
+                except:
+                    model_data['semantic_categories'] = {}
+                
+                # Parse grammar rules
+                try:
+                    if len(metadata_parts) > 3:
+                        model_data['grammar_rules'] = eval(metadata_parts[3]) if metadata_parts[3] != '[]' else []
+                    else:
+                        model_data['grammar_rules'] = []
+                except:
+                    model_data['grammar_rules'] = []
+                
+                # Extract pattern data (skip metadata row)
+                pattern_df = df.iloc[1:].copy()
+                
+                model_data['patterns'] = [p for p in pattern_df['patterns'].tolist() if p]
+                model_data['responses'] = [r for r in pattern_df['responses'].tolist() if r]
+                model_data['weights'] = pattern_df['weights'].tolist()
+                
+                # Parse response templates
+                response_templates = []
+                for template_str in pattern_df['response_templates'].tolist():
+                    try:
+                        template = eval(template_str) if template_str not in ['{}', ''] else {}
+                        response_templates.append(template)
+                    except:
+                        response_templates.append({})
+                
+                model_data['response_templates'] = response_templates
+                
+            else:
+                # Fallback: Load old format (for backward compatibility)
+                model_data = {
+                    'patterns': [p for p in df['patterns'].tolist() if p],
+                    'responses': [r for r in df['responses'].tolist() if r],
+                    'weights': df['weights'].tolist(),
+                    'confidence': df.get('confidence', pd.Series([0.5])).iloc[0] if 'confidence' in df.columns else 0.5,
+                    'keywords': df.get('keywords', pd.Series([''])).iloc[0].split() if 'keywords' in df.columns else [],
+                    'training_samples': df.get('training_samples', pd.Series([0])).iloc[0] if 'training_samples' in df.columns else 0,
+                    'model_id': model_id,
+                    'knowledge_base': {},
+                    'semantic_categories': {},
+                    'response_templates': [],
+                    'grammar_rules': []
+                }
+            
+            return model_data
+            
+        except Exception as e:
+            print(f"Error loading optimized model {model_id}: {e}")
+            return None
+    
+    def load_all_models(self) -> List[Dict[str, Any]]:
+        models = []
+        
+        if not os.path.exists(self.models_dir):
+            return models
+        
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                try:
+                    model_id = int(filename.split("_")[2].split(".")[0])
+                    model = self.load_mini_model(model_id)
+                    if model:
+                        models.append(model)
+                except (ValueError, IndexError):
+                    print(f"Warning: Invalid model filename format: {filename}")
+                    continue
+        
+        return models
+    
+    def get_model_count(self) -> int:
+        if not os.path.exists(self.models_dir):
+            return 0
+        
+        count = 0
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                count += 1
+        
+        return count
+    
+    def get_next_model_id(self) -> int:
+        if not os.path.exists(self.models_dir):
+            return 1
+        
+        max_id = 0
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                try:
+                    model_id = int(filename.split("_")[2].split(".")[0])
+                    max_id = max(max_id, model_id)
+                except (ValueError, IndexError):
+                    continue
+        
+        return max_id + 1
+    
+    def delete_model(self, model_id: int) -> bool:
+        filename = f"AgGPT_Expert_{model_id:04d}.feather"
+        filepath = os.path.join(self.models_dir, filename)
+        
+        if os.path.exists(filepath):
+            try:
+                os.remove(filepath)
+                print(f"Deleted model: {filename}")
+                return True
+            except Exception as e:
+                print(f"Error deleting model {model_id}: {e}")
+                return False
+        
+        return False
+    
+    def clear_all_models(self) -> int:
+        if not os.path.exists(self.models_dir):
+            return 0
+        
+        deleted_count = 0
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                try:
+                    os.remove(os.path.join(self.models_dir, filename))
+                    deleted_count += 1
+                except Exception as e:
+                    print(f"Error deleting {filename}: {e}")
+        
+        print(f"Deleted {deleted_count} model files")
+        return deleted_count
+
+
+def similarity_score(text1: str, text2: str) -> float:
+    """Enhanced semantic similarity calculation for AgGPT-19"""
+    if not text1 or not text2:
+        return 0.0
+    
+    # Normalize texts
+    text1_clean = text1.lower().strip()
+    text2_clean = text2.lower().strip()
+    
+    # Exact match bonus
+    if text1_clean == text2_clean:
+        return 1.0
+    
+    # Character-level similarity (for typos and variations)
+    char_sim = _character_similarity(text1_clean, text2_clean)
+    
+    # Word-level analysis
+    words1 = set(text1_clean.split())
+    words2 = set(text2_clean.split())
+    
+    if not words1 or not words2:
+        return char_sim * 0.3
+    
+    # Jaccard similarity (word overlap)
+    intersection = len(words1.intersection(words2))
+    union = len(words1.union(words2))
+    jaccard = intersection / union if union > 0 else 0.0
+    
+    # Semantic word analysis
+    semantic_sim = _semantic_word_similarity(words1, words2)
+    
+    # N-gram similarity
+    ngram_sim = _ngram_similarity(text1_clean, text2_clean)
+    
+    # Length penalty for very different lengths
+    len1, len2 = len(text1_clean.split()), len(text2_clean.split())
+    length_penalty = 1.0 - min(abs(len1 - len2) / max(len1, len2, 1), 0.5)
+    
+    # Combine all similarity measures
+    final_score = (
+        jaccard * 0.4 +           # Word overlap
+        semantic_sim * 0.3 +      # Semantic similarity
+        ngram_sim * 0.2 +         # Character patterns
+        char_sim * 0.1            # Character similarity
+    ) * length_penalty
+    
+    return min(final_score, 1.0)
+
+
+def _character_similarity(text1: str, text2: str) -> float:
+    """Calculate character-level similarity using longest common subsequence"""
+    if not text1 or not text2:
+        return 0.0
+    
+    # Simple LCS implementation
+    len1, len2 = len(text1), len(text2)
+    dp = [[0] * (len2 + 1) for _ in range(len1 + 1)]
+    
+    for i in range(1, len1 + 1):
+        for j in range(1, len2 + 1):
+            if text1[i-1] == text2[j-1]:
+                dp[i][j] = dp[i-1][j-1] + 1
+            else:
+                dp[i][j] = max(dp[i-1][j], dp[i][j-1])
+    
+    lcs_length = dp[len1][len2]
+    return (2.0 * lcs_length) / (len1 + len2) if (len1 + len2) > 0 else 0.0
+
+
+def _semantic_word_similarity(words1: set, words2: set) -> float:
+    """Calculate semantic similarity between word sets"""
+    if not words1 or not words2:
+        return 0.0
+    
+    # Common semantic patterns
+    synonyms = {
+        'hello': {'hi', 'hey', 'greetings', 'good morning', 'good afternoon'},
+        'thanks': {'thank you', 'appreciate', 'grateful'},
+        'yes': {'yeah', 'yep', 'sure', 'absolutely', 'definitely'},
+        'no': {'nope', 'negative', 'not really'},
+        'good': {'great', 'excellent', 'wonderful', 'amazing', 'fantastic'},
+        'bad': {'terrible', 'awful', 'horrible', 'poor'},
+        'big': {'large', 'huge', 'enormous', 'massive'},
+        'small': {'little', 'tiny', 'mini', 'minute'},
+    }
+    
+    # Find semantic matches
+    semantic_matches = 0
+    total_comparisons = 0
+    
+    for word1 in words1:
+        for word2 in words2:
+            total_comparisons += 1
+            
+            # Direct match
+            if word1 == word2:
+                semantic_matches += 1
+                continue
+            
+            # Check synonyms
+            for key, synonym_set in synonyms.items():
+                if word1 in synonym_set and word2 in synonym_set:
+                    semantic_matches += 0.8
+                    break
+                elif (word1 == key and word2 in synonym_set) or (word2 == key and word1 in synonym_set):
+                    semantic_matches += 0.9
+                    break
+            
+            # Partial word matching (for variations)
+            if len(word1) > 3 and len(word2) > 3:
+                if word1 in word2 or word2 in word1:
+                    semantic_matches += 0.6
+                elif word1[:3] == word2[:3]:  # Same prefix
+                    semantic_matches += 0.4
+    
+    return semantic_matches / total_comparisons if total_comparisons > 0 else 0.0
+
+
+def _ngram_similarity(text1: str, text2: str, n: int = 3) -> float:
+    """Calculate n-gram similarity for character patterns"""
+    if len(text1) < n or len(text2) < n:
+        return 0.0
+    
+    ngrams1 = set(text1[i:i+n] for i in range(len(text1) - n + 1))
+    ngrams2 = set(text2[i:i+n] for i in range(len(text2) - n + 1))
+    
+    if not ngrams1 or not ngrams2:
+        return 0.0
+    
+    intersection = len(ngrams1.intersection(ngrams2))
+    union = len(ngrams1.union(ngrams2))
+    
+    return intersection / union if union > 0 else 0.0
+
+
+def calculate_confidence_score(patterns: List[str], responses: List[str]) -> float:
+    if not patterns or not responses or len(patterns) != len(responses):
+        return 0.1
+    
+    base_confidence = min(0.9, len(patterns) / 10.0)
+    
+    return max(0.1, min(1.0, base_confidence))
+
+
+if __name__ == "__main__":
+    manager = FeatherManager()
+    
+    test_model = {
+        'patterns': ['hello', 'hi', 'hey'],
+        'responses': ['Hello! How can I help you?', 'Hi there!', 'Hey! What\'s up?'],
+        'weights': [1.0, 0.9, 0.8],
+        'confidence': 0.8,
+        'grammar_rules': ['capitalize_first_word', 'end_with_punctuation'],
+        'keywords': ['greeting', 'hello', 'hi'],
+        'training_samples': 150
+    }
+    
+    model_id = manager.get_next_model_id()
+    manager.save_mini_model(test_model, model_id)
+    loaded_model = manager.load_mini_model(model_id)
+    
+    print(f"Original model: {test_model}")
+    print(f"Loaded model: {loaded_model}")
+    print(f"Models count: {manager.get_model_count()}")

train.py

@@ -0,0 +1,690 @@
+import os
+import re
+import random
+import math
+import yaml
+from typing import List, Dict, Tuple, Set, Any
+from collections import defaultdict, Counter
+import pandas as pd
+from tqdm import tqdm
+
+from feather import FeatherManager, similarity_score, calculate_confidence_score
+
+class GrammarRules:
+    @staticmethod
+    def apply_all_rules(text: str) -> str:
+        if not text:
+            return text
+        
+        return text.strip()
+
+class PatternExtractor:
+    
+    def __init__(self):
+        # Enhanced semantic patterns for AgGPT-19
+        self.semantic_groups = {
+            'questions': ['what', 'how', 'why', 'when', 'where', 'who', 'which', 'can', 'could', 'would', 'should', 'is', 'are', 'do', 'does'],
+            'greetings': ['hello', 'hi', 'hey', 'greetings', 'good morning', 'good afternoon', 'good evening'],
+            'farewells': ['goodbye', 'bye', 'see you', 'farewell', 'take care'],
+            'requests': ['please', 'can you', 'could you', 'would you', 'help me', 'i need', 'i want'],
+            'emotions': ['happy', 'sad', 'angry', 'excited', 'worried', 'confused', 'frustrated'],
+            'affirmations': ['yes', 'yeah', 'sure', 'okay', 'alright', 'definitely', 'absolutely'],
+            'negations': ['no', 'not', 'never', 'nothing', 'none', 'neither'],
+        }
+        
+        self.stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'}
+    
+    def extract_keywords(self, text: str) -> List[str]:
+        if not text:
+            return []
+        
+        full_text_normalized = re.sub(r'\s+', ' ', text.strip().lower())
+        words = re.findall(r'\b[a-zA-Z]+\b', full_text_normalized)
+        
+        # Remove stop words for better keyword extraction
+        meaningful_words = [word for word in words if word not in self.stop_words and len(word) > 2]
+        
+        # Add semantic categories
+        semantic_keywords = []
+        for category, category_words in self.semantic_groups.items():
+            if any(word in meaningful_words for word in category_words):
+                semantic_keywords.append(f'semantic_{category}')
+        
+        # Extract named entities (simple approach)
+        entities = self._extract_simple_entities(full_text_normalized)
+        
+        result = [full_text_normalized]
+        result.extend(meaningful_words[:10])  # Limit to top 10 words
+        result.extend(semantic_keywords)
+        result.extend(entities)
+        
+        return list(set(result))
+    
+    def _extract_simple_entities(self, text: str) -> List[str]:
+        """Extract simple entities without external libraries"""
+        entities = []
+        
+        # Numbers
+        numbers = re.findall(r'\b\d+\b', text)
+        entities.extend([f'number_{num}' for num in numbers[:3]])
+        
+        # Capitalized words (potential names/places)
+        original_words = re.findall(r'\b[A-Z][a-z]+\b', text)
+        entities.extend([f'entity_{word.lower()}' for word in original_words[:3]])
+        
+        # Time expressions
+        time_patterns = ['today', 'tomorrow', 'yesterday', 'morning', 'evening', 'night', 'afternoon']
+        for pattern in time_patterns:
+            if pattern in text.lower():
+                entities.append(f'time_{pattern}')
+        
+        return entities
+    
+    def create_pattern(self, user_input: str) -> str:
+        if not user_input:
+            return ""
+        
+        # Enhanced pattern creation with semantic understanding
+        normalized = re.sub(r'\s+', ' ', user_input.strip().lower())
+        
+        # Extract semantic structure
+        words = normalized.split()
+        semantic_pattern = []
+        
+        for word in words:
+            # Check if word belongs to semantic groups
+            added_semantic = False
+            for category, category_words in self.semantic_groups.items():
+                if word in category_words:
+                    semantic_pattern.append(f'<{category}>')
+                    added_semantic = True
+                    break
+            
+            if not added_semantic:
+                if word in self.stop_words:
+                    semantic_pattern.append(f'<stop>')
+                elif word.isdigit():
+                    semantic_pattern.append('<number>')
+                elif len(word) > 6:  # Longer words are more specific
+                    semantic_pattern.append(word)
+                else:
+                    semantic_pattern.append(f'<word>')
+        
+        # Create both literal and semantic patterns
+        literal_pattern = f" {normalized} "
+        semantic_structure = " ".join(semantic_pattern)
+        
+        return f"{literal_pattern}|{semantic_structure}"
+    
+    def calculate_pattern_similarity(self, pattern1: str, pattern2: str) -> float:
+        # Enhanced similarity calculation for AgGPT-19
+        if not pattern1 or not pattern2:
+            return 0.0
+            
+        # Split patterns if they contain semantic structure
+        parts1 = pattern1.strip().split('|')
+        parts2 = pattern2.strip().split('|')
+        
+        literal1 = parts1[0].strip()
+        literal2 = parts2[0].strip()
+        
+        # Calculate literal similarity
+        literal_sim = similarity_score(literal1, literal2)
+        
+        # Calculate semantic similarity if available
+        semantic_sim = 0.0
+        if len(parts1) > 1 and len(parts2) > 1:
+            semantic1 = parts1[1].strip()
+            semantic2 = parts2[1].strip()
+            semantic_sim = self._semantic_structure_similarity(semantic1, semantic2)
+        
+        # Combine similarities
+        if semantic_sim > 0:
+            return (literal_sim * 0.7 + semantic_sim * 0.3)
+        else:
+            return literal_sim
+    
+    def _semantic_structure_similarity(self, struct1: str, struct2: str) -> float:
+        """Compare semantic structures"""
+        if not struct1 or not struct2:
+            return 0.0
+        
+        tokens1 = struct1.split()
+        tokens2 = struct2.split()
+        
+        if not tokens1 or not tokens2:
+            return 0.0
+        
+        # Compare token patterns
+        matches = 0
+        total = max(len(tokens1), len(tokens2))
+        
+        for i in range(min(len(tokens1), len(tokens2))):
+            if tokens1[i] == tokens2[i]:
+                matches += 1
+            elif tokens1[i].startswith('<') and tokens2[i].startswith('<'):
+                # Both are semantic tokens, partial match
+                matches += 0.5
+        
+        return matches / total if total > 0 else 0.0
+
+
+class MiniModelTrainer:
+    
+    def __init__(self, feather_manager: FeatherManager):
+        self.feather_manager = feather_manager
+        self.pattern_extractor = PatternExtractor()
+        self.grammar_rules = GrammarRules()
+    
+    def train_mini_model(self, training_pairs: List[Tuple[str, str]], confidence_threshold: float = 0.1) -> Dict[str, Any]:
+        if not training_pairs or len(training_pairs) < 2:
+            return None
+
+        # Enhanced training for AgGPT-19
+        keyword_patterns = []
+        responses = []
+        pattern_confidences = []
+        all_keywords = []
+        response_templates = []
+        knowledge_base = {}
+        
+        for user_input, ai_response in training_pairs:
+            processed_response = ai_response.strip()
+            
+            # Extract both patterns and semantic understanding
+            pattern = self.pattern_extractor.create_pattern(user_input)
+            keywords = self.pattern_extractor.extract_keywords(user_input)
+            all_keywords.extend(keywords)
+            
+            # Create response templates for generation
+            template = self._create_response_template(ai_response, user_input)
+            response_templates.append(template)
+            
+            # Build knowledge base
+            knowledge_entry = self._extract_knowledge(user_input, ai_response)
+            if knowledge_entry:
+                knowledge_base.update(knowledge_entry)
+            
+            # Add the main pattern
+            keyword_patterns.append(pattern)
+            responses.append(processed_response)
+            individual_confidence = min(0.9, len(training_pairs) / 20.0)
+            pattern_confidences.append(individual_confidence)
+        
+        if not keyword_patterns:
+            return None
+        
+        base_confidence = min(0.9, len(training_pairs) / 20.0)
+        keyword_counter = Counter(all_keywords)
+        top_keywords = [word for word, count in keyword_counter.most_common(15)]
+        
+        # Enhanced mini-model structure for AgGPT-19
+        mini_model = {
+            'patterns': keyword_patterns,
+            'responses': responses,
+            'response_templates': response_templates,
+            'knowledge_base': knowledge_base,
+            'pattern_confidences': pattern_confidences,
+            'confidence': base_confidence,
+            'grammar_rules': [],
+            'keywords': top_keywords,
+            'training_samples': len(training_pairs),
+            'semantic_categories': self._analyze_semantic_categories(training_pairs)
+        }
+        
+        return mini_model
+    
+    def _create_response_template(self, response: str, input_text: str) -> Dict[str, Any]:
+        """Create a template for generating similar responses"""
+        # Extract placeholders and structure
+        template = {
+            'structure': 'direct',  # direct, question, explanation, list
+            'length': 'medium',     # short, medium, long
+            'tone': 'neutral',      # friendly, formal, casual, neutral
+            'placeholders': [],
+            'key_phrases': [],
+        }
+        
+        words = response.split()
+        
+        # Determine response structure
+        if '?' in response:
+            template['structure'] = 'question'
+        elif any(word in response.lower() for word in ['first', 'second', 'then', 'next', '1.', '2.']):
+            template['structure'] = 'list'
+        elif len(words) > 50:
+            template['structure'] = 'explanation'
+        
+        # Determine length
+        if len(words) < 10:
+            template['length'] = 'short'
+        elif len(words) > 30:
+            template['length'] = 'long'
+        
+        # Determine tone
+        if any(word in response.lower() for word in ['please', 'thank you', 'great', 'wonderful']):
+            template['tone'] = 'friendly'
+        elif any(word in response.lower() for word in ['hey', 'yeah', 'cool', 'awesome']):
+            template['tone'] = 'casual'
+        
+        # Extract key phrases (simple approach)
+        sentences = response.split('.')
+        template['key_phrases'] = [sent.strip() for sent in sentences if sent.strip() and len(sent.strip()) > 10][:3]
+        
+        return template
+    
+    def _extract_knowledge(self, question: str, answer: str) -> Dict[str, str]:
+        """Extract knowledge facts from Q&A pairs"""
+        knowledge = {}
+        
+        # Simple fact extraction
+        question_lower = question.lower()
+        
+        # Extract definitions
+        if any(word in question_lower for word in ['what is', 'what are', 'define']):
+            subject = self._extract_subject(question)
+            if subject:
+                knowledge[f'definition_{subject}'] = answer[:200]  # Limit length
+        
+        # Extract how-to knowledge
+        elif 'how to' in question_lower or 'how do' in question_lower:
+            topic = question_lower.replace('how to', '').replace('how do', '').strip()
+            if topic:
+                knowledge[f'howto_{topic[:20]}'] = answer[:300]
+        
+        # Extract factual knowledge
+        elif any(word in question_lower for word in ['where', 'when', 'who', 'which']):
+            knowledge[f'fact_{hash(question) % 10000}'] = answer[:150]
+        
+        return knowledge
+    
+    def _extract_subject(self, question: str) -> str:
+        """Extract the main subject from a question"""
+        words = question.lower().split()
+        
+        # Remove question words
+        question_words = {'what', 'is', 'are', 'the', 'a', 'an'}
+        filtered_words = [word for word in words if word not in question_words]
+        
+        if filtered_words:
+            return '_'.join(filtered_words[:3])  # Take first 3 meaningful words
+        
+        return ''
+    
+    def _analyze_semantic_categories(self, training_pairs: List[Tuple[str, str]]) -> Dict[str, int]:
+        """Analyze what types of conversations this model handles"""
+        categories = {
+            'questions': 0, 'greetings': 0, 'requests': 0, 'explanations': 0,
+            'personal': 0, 'technical': 0, 'casual': 0, 'factual': 0
+        }
+        
+        for user_input, ai_response in training_pairs:
+            input_lower = user_input.lower()
+            
+            # Categorize inputs
+            if any(word in input_lower for word in ['what', 'how', 'why', 'when', 'where']):
+                categories['questions'] += 1
+            if any(word in input_lower for word in ['hello', 'hi', 'hey']):
+                categories['greetings'] += 1
+            if any(word in input_lower for word in ['please', 'can you', 'help']):
+                categories['requests'] += 1
+            if any(word in input_lower for word in ['i', 'my', 'me']):
+                categories['personal'] += 1
+            if any(word in input_lower for word in ['code', 'program', 'technical', 'computer']):
+                categories['technical'] += 1
+            if len(ai_response.split()) > 30:
+                categories['explanations'] += 1
+        
+        return categories
+    
+    def should_merge_models(self, model1: Dict[str, Any], model2: Dict[str, Any], merge_threshold: float = 0.8) -> bool:
+        keywords1 = set(model1.get('keywords', []))
+        keywords2 = set(model2.get('keywords', []))
+        
+        if not keywords1 or not keywords2:
+            return False
+        
+        keyword_similarity = len(keywords1.intersection(keywords2)) / len(keywords1.union(keywords2))
+        
+        responses1 = model1.get('responses', [])
+        responses2 = model2.get('responses', [])
+        
+        response_similarities = []
+        for r1 in responses1[:5]:
+            for r2 in responses2[:5]:
+                sim = similarity_score(r1, r2)
+                response_similarities.append(sim)
+        
+        avg_response_similarity = sum(response_similarities) / len(response_similarities) if response_similarities else 0
+        
+        min_confidence = min(model1.get('confidence', 0), model2.get('confidence', 0))
+        
+        return (keyword_similarity > merge_threshold and 
+                avg_response_similarity > merge_threshold and 
+                min_confidence > 0.7)
+    
+    def merge_mini_models(self, model1: Dict[str, Any], model2: Dict[str, Any]) -> Dict[str, Any]:
+        patterns1 = model1.get('patterns', [])
+        patterns2 = model2.get('patterns', [])
+        responses1 = model1.get('responses', [])
+        responses2 = model2.get('responses', [])
+        confidences1 = model1.get('pattern_confidences', [1.0] * len(patterns1))
+        confidences2 = model2.get('pattern_confidences', [1.0] * len(patterns2))
+        
+        merged_model = {
+            'patterns': patterns1 + patterns2,
+            'responses': responses1 + responses2,
+            'pattern_confidences': confidences1 + confidences2,
+            'confidence': (model1.get('confidence', 0) + model2.get('confidence', 0)) / 2,
+            'grammar_rules': list(set(model1.get('grammar_rules', []) + model2.get('grammar_rules', []))),
+            'keywords': list(set(model1.get('keywords', []) + model2.get('keywords', []))),
+            'training_samples': model1.get('training_samples', 0) + model2.get('training_samples', 0)
+        }
+        
+        return merged_model
+
+
+class AgGPTTrainer:
+    
+    def __init__(self, models_dir: str = "models"):
+        self.feather_manager = FeatherManager(models_dir)
+        self.mini_trainer = MiniModelTrainer(self.feather_manager)
+        self.target_size_mb = 5
+        self.estimated_size_per_pair = 1000
+        self.chunk_size = (self.target_size_mb * 1024 * 1024) // self.estimated_size_per_pair
+        self.readable_weights_dir = "readable_weights"
+        os.makedirs(self.readable_weights_dir, exist_ok=True)
+    
+    def save_model_as_yaml(self, model_data: Dict[str, Any], model_id: int):
+        try:
+            filename = f"AgGPT_Model_{model_id:04d}.yaml"
+            filepath = os.path.join(self.readable_weights_dir, filename)
+            
+            print(f"Creating YAML data for model {model_id}...")
+            yaml_data = {
+                'model_info': {
+                    'model_id': model_id,
+                    'confidence': model_data.get('confidence', 0.5),
+                    'training_samples': model_data.get('training_samples', 0),
+                    'keywords': model_data.get('keywords', [])
+                },
+                'patterns_and_responses': []
+            }
+            
+            patterns = model_data.get('patterns', [])
+            responses = model_data.get('responses', [])
+            weights = model_data.get('weights', [])
+            
+            print(f"Processing {len(patterns)} patterns...")
+            for i in range(len(patterns)):
+                entry = {
+                    'pattern': patterns[i] if i < len(patterns) else '',
+                    'response': responses[i] if i < len(responses) else '',
+                    'weight': weights[i] if i < len(weights) else 1.0
+                }
+                yaml_data['patterns_and_responses'].append(entry)
+            
+            print(f"Writing YAML to {filepath}...")
+            with open(filepath, 'w', encoding='utf-8') as f:
+                yaml.dump(yaml_data, f, default_flow_style=False, allow_unicode=True, indent=2)
+            
+            print(f"Saved readable model: {filename}")
+        except Exception as e:
+            print(f"Error in save_model_as_yaml: {e}")
+            import traceback
+            traceback.print_exc()
+    
+    def load_training_data(self, file_path: str) -> List[Tuple[str, str]]:
+        training_pairs = []
+        
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        
+        conversations = content.split('<eos>')
+        
+        print(f"Processing {len(conversations)} conversation chunks...")
+        
+        for conversation in tqdm(conversations, desc="Parsing conversations"):
+            conversation = conversation.strip()
+            if not conversation:
+                continue
+            
+            user_match = re.search(r'user:\s*(.*?)(?=\n<pad>|\nai:|$)', conversation, re.DOTALL)
+            ai_match = re.search(r'ai:\s*(.*?)$', conversation, re.DOTALL)
+            
+            if user_match and ai_match:
+                user_input = user_match.group(1).strip()
+                ai_response = ai_match.group(1).strip()
+                
+                user_input = re.sub(r'<pad>', '', user_input).strip()
+                ai_response = re.sub(r'<pad>', '', ai_response).strip()
+                
+                if user_input and ai_response and len(user_input) > 0 and len(ai_response) > 0:
+                    training_pairs.append((user_input, ai_response))
+        
+        print(f"Extracted {len(training_pairs)} training pairs")
+        return training_pairs
+    
+    def create_training_chunks(self, training_pairs: List[Tuple[str, str]]) -> List[List[Tuple[str, str]]]:
+        shuffled_pairs = training_pairs.copy()
+        random.shuffle(shuffled_pairs)
+        
+        chunks = []
+        total_pairs = len(shuffled_pairs)
+        
+        for i in range(0, total_pairs, self.chunk_size):
+            chunk = shuffled_pairs[i:i + self.chunk_size]
+            if len(chunk) >= 5:
+                chunks.append(chunk)
+        
+        print(f"Created {len(chunks)} training chunks (target: {self.target_size_mb}MB each)")
+        return chunks
+    
+    def train_multiple_corpora(self, training_files: List[str] = None, merge_similar: bool = True):
+        """Train on multiple corpora files sequentially"""
+        if training_files is None:
+            # Automatically find all text files in training_corpora directory
+            training_dir = "training_corpora"
+            if os.path.exists(training_dir):
+                training_files = []
+                for filename in sorted(os.listdir(training_dir)):
+                    if filename.endswith('.txt'):
+                        training_files.append(os.path.join(training_dir, filename))
+                print(f"Found {len(training_files)} text files in {training_dir}")
+            else:
+                print(f"Warning: {training_dir} directory not found, falling back to default files")
+                training_files = ["training_data/corpora.txt", "training_data/corpora2.txt"]
+        
+        print("Starting AgGPT-19 Multi-Corpora Training with Enhanced Intelligence")
+        print("=" * 70)
+        
+        cleared_count = self.feather_manager.clear_all_models()
+        if cleared_count > 0:
+            print(f"Cleared {cleared_count} existing models")
+        
+        all_trained_models = []
+        total_model_id = 1
+        
+        for file_idx, training_file in enumerate(training_files, 1):
+            print(f"\n--- Training on file {file_idx}/{len(training_files)}: {training_file} ---")
+            
+            if not os.path.exists(training_file):
+                print(f"Warning: Training file {training_file} does not exist. Skipping...")
+                continue
+            
+            if os.path.getsize(training_file) == 0:
+                print(f"Warning: Training file {training_file} is empty. Skipping...")
+                continue
+            
+            print(f"Loading training data from {training_file}...")
+            training_pairs = self.load_training_data(training_file)
+            
+            if not training_pairs:
+                print(f"No training data found in {training_file}. Skipping...")
+                continue
+            
+            print(f"Creating training chunks for {training_file}...")
+            training_chunks = self.create_training_chunks(training_pairs)
+            
+            print(f"Training mini-models from {training_file}...")
+            file_trained_models = []
+            
+            progress_bar = tqdm(training_chunks, desc=f"Training from {os.path.basename(training_file)}")
+            for chunk_idx, chunk in enumerate(progress_bar):
+                print(f"\nProcessing chunk {chunk_idx + 1}/{len(training_chunks)}")
+                mini_model = self.mini_trainer.train_mini_model(chunk)
+                
+                if mini_model:
+                    file_trained_models.append(mini_model)
+                    all_trained_models.append(mini_model)
+                    print(f"Saving model {total_model_id}...")
+                    self.feather_manager.save_mini_model(mini_model, total_model_id)
+                    
+                    if total_model_id == 1:
+                        print("Saving first model as YAML...")
+                        try:
+                            self.save_model_as_yaml(mini_model, total_model_id)
+                            print("YAML saved successfully")
+                        except Exception as e:
+                            print(f"Error saving YAML: {e}")
+                    
+                    total_model_id += 1
+                    print(f"Model {total_model_id - 1} completed")
+                    
+                    try:
+                        progress_bar.set_postfix({
+                            'File Models': len(file_trained_models),
+                            'Total Models': len(all_trained_models),
+                            'Confidence': f"{mini_model['confidence']:.3f}"
+                        })
+                    except Exception as e:
+                        print(f"Error updating progress bar: {e}")
+            
+            print(f"Completed training on {training_file}: {len(file_trained_models)} mini-models created")
+            print(f"Total models so far: {len(all_trained_models)}")
+            
+            if merge_similar and len(all_trained_models) > 1:
+                print(f"Merging similar models after processing {training_file}...")
+                self._merge_similar_models()
+                current_count = self.feather_manager.get_model_count()
+                print(f"Models after merging: {current_count}")
+        
+        print(f"\n--- Multi-Corpora Training Complete ---")
+        final_count = self.feather_manager.get_model_count()
+        print(f"Final model count: {final_count}")
+        print(f"Trained on {len([f for f in training_files if os.path.exists(f) and os.path.getsize(f) > 0])} corpora files")
+        print("=" * 70)
+    
+    def train(self, training_file: str = "training_data/corpora.txt", merge_similar: bool = True):
+        print("Starting AgGPT-19 Training with Enhanced Intelligence")
+        print("=" * 60)
+        
+        cleared_count = self.feather_manager.clear_all_models()
+        if cleared_count > 0:
+            print(f"Cleared {cleared_count} existing models")
+        
+        print("Loading training data...")
+        training_pairs = self.load_training_data(training_file)
+        
+        if not training_pairs:
+            print("No training data found!")
+            return
+        
+        print("Creating training chunks...")
+        training_chunks = self.create_training_chunks(training_pairs)
+        
+        print("Training mini-models...")
+        trained_models = []
+        model_id = 1
+        
+        progress_bar = tqdm(training_chunks, desc="Training mini-models")
+        for chunk in progress_bar:
+            mini_model = self.mini_trainer.train_mini_model(chunk)
+            
+            if mini_model:
+                trained_models.append(mini_model)
+                self.feather_manager.save_mini_model(mini_model, model_id)
+                
+                if model_id == 1:
+                    self.save_model_as_yaml(mini_model, model_id)
+                
+                model_id += 1
+                
+                progress_bar.set_postfix({
+                    'Models': len(trained_models),
+                    'Confidence': f"{mini_model['confidence']:.3f}"
+                })
+        
+        print(f"Trained {len(trained_models)} mini-models")
+        
+        if merge_similar and len(trained_models) > 1:
+            print("Merging similar models...")
+            self._merge_similar_models()
+        
+        final_count = self.feather_manager.get_model_count()
+        print(f"Training complete! Final model count: {final_count}")
+        print("=" * 60)
+    
+    def _merge_similar_models(self):
+        all_models = self.feather_manager.load_all_models()
+        if len(all_models) < 2:
+            return
+        
+        merged_pairs = []
+        models_to_delete = set()
+        
+        print(f"Checking {len(all_models)} models for merging opportunities...")
+        
+        progress_bar = tqdm(range(len(all_models)), desc="Merging models")
+        for i in progress_bar:
+            if i in models_to_delete:
+                continue
+            
+            for j in range(i + 1, len(all_models)):
+                if j in models_to_delete:
+                    continue
+                
+                model1 = all_models[i]
+                model2 = all_models[j]
+                
+                if self.mini_trainer.should_merge_models(model1, model2):
+                    merged_model = self.mini_trainer.merge_mini_models(model1, model2)
+                    
+                    new_id = self.feather_manager.get_next_model_id()
+                    self.feather_manager.save_mini_model(merged_model, new_id)
+                    
+                    models_to_delete.add(i)
+                    models_to_delete.add(j)
+                    merged_pairs.append((model1.get('model_id', i), model2.get('model_id', j), new_id))
+                    
+                    break
+        
+        for model_idx in models_to_delete:
+            if model_idx < len(all_models):
+                model_id = all_models[model_idx].get('model_id', model_idx + 1)
+                self.feather_manager.delete_model(model_id)
+        
+        if merged_pairs:
+            print(f"Merged {len(merged_pairs)} pairs of similar models")
+        else:
+            print("No similar models found for merging")
+
+
+def main():
+    print("AgGPT-19 Enhanced Intelligence Trainer")
+    print("=" * 50)
+    
+    trainer = AgGPTTrainer()
+    
+    try:
+        trainer.train_multiple_corpora(merge_similar=False)  # DISABLED: merging takes too long
+    except KeyboardInterrupt:
+        print("\nTraining interrupted by user")
+    except Exception as e:
+        print(f"Training failed: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()