Upload 7 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+banner.png filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 AG
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,26 @@
----
-license: mit
----
+---  
+license: mit  
+language:  
+- en  
+---
+
+<img src="banner.png" alt="AgGPT-20 Banner" width="100%">
+
+# AgGPT-20
+
+**AgGPT-20** is a revolutionary modular AI training framework using Scalable Feather Architecture that automatically discovers and trains on all `.txt` files in `training_corpora/`, creating specialized mini-models stored as `.feather` files in the `models/` directory. The system is extremely modular - you can train individual models using `target_train.py`, manually add `.feather` files to the models folder from other sources, or use the full multi-corpora trainer with `train.py`, and all models will be automatically discovered and used by the chat interface. Install dependencies with `pip install pandas pyarrow tqdm pyyaml`, place training data in the proper format (`user: [input]\n<pad>\nai: [response]\n<eos>`) in `training_corpora/` or `extra_corpora/`, run `python train.py` for full training or `python target_train.py` for single-file training, then chat with `python chat.py`. The architecture enables lightning-fast loading, cross-platform compatibility, and seamless model distribution - just drop any compatible `.feather` model into the models folder and it's instantly available.
+
+## Quick Start
+```bash
+# Install dependencies
+pip install pandas pyarrow tqdm pyyaml
+
+# Train on all corpora files
+python train.py
+
+# Or train on a single target file
+python target_train.py
+
+# Start chatting
+python chat.py
+```

chat.py

@@ -0,0 +1,318 @@
+import os
+import re
+import random
+from typing import List, Dict, Tuple, Any
+from collections import defaultdict
+import math
+from feather import FeatherManager, similarity_score
+from train import GrammarRules, PatternExtractor
+
+class ResponseGenerator: 
+    def __init__(self, feather_manager: FeatherManager):
+        self.feather_manager = feather_manager
+        self.pattern_extractor = PatternExtractor()
+        self.grammar_rules = GrammarRules()
+        self.models = []
+        self.context_window = []
+        self.max_context_length = 10
+        
+    def load_models(self):
+        print("Loading mini-models...")
+        self.models = self.feather_manager.load_all_models()
+        print(f"Loaded {len(self.models)} mini-models")
+        
+        if not self.models:
+            print("No trained models found! Please run train.py first.")
+            return False
+        
+        return True
+    
+    def calculate_model_scores(self, user_input: str) -> List[Tuple[Dict[str, Any], float]]:
+        if not self.models:
+            return []
+        
+        input_pattern = self.pattern_extractor.create_pattern(user_input)
+        input_keywords = set(self.pattern_extractor.extract_keywords(user_input))
+        
+        model_scores = []
+        
+        for model in self.models:
+            score = 0.0
+            pattern_matches = 0
+            keyword_matches = 0
+            
+            for pattern in model.get('patterns', []):
+                pattern_sim = self.pattern_extractor.calculate_pattern_similarity(input_pattern, pattern)
+                score += pattern_sim
+                if pattern_sim > 0.3:
+                    pattern_matches += 1
+            
+            model_keywords = set(model.get('keywords', []))
+            if model_keywords and input_keywords:
+                keyword_overlap = len(input_keywords.intersection(model_keywords))
+                keyword_total = len(input_keywords.union(model_keywords))
+                keyword_score = keyword_overlap / keyword_total if keyword_total > 0 else 0
+                score += keyword_score * 2
+                keyword_matches = keyword_overlap
+            
+            confidence = model.get('confidence', 0.5)
+            score *= confidence
+            
+            training_samples = model.get('training_samples', 1)
+            training_bonus = min(0.2, training_samples / 100)
+            score += training_bonus
+            
+            context_bonus = self._calculate_context_bonus(user_input, model)
+            score += context_bonus
+            
+            model_scores.append((model, score))
+        
+        model_scores.sort(key=lambda x: x[1], reverse=True)
+        
+        return model_scores
+    
+    def _calculate_context_bonus(self, user_input: str, model: Dict[str, Any]) -> float:
+        if not self.context_window:
+            return 0.0
+        
+        context_bonus = 0.0
+        
+        for prev_input, prev_response in self.context_window[-3:]:
+            for pattern in model.get('patterns', [])[:5]:
+                pattern_sim = similarity_score(prev_input, pattern.strip())
+                context_bonus += pattern_sim * 0.1
+        
+        return min(context_bonus, 0.3)
+    
+    def select_top_models(self, model_scores: List[Tuple[Dict[str, Any], float]], top_k: int = 5) -> List[Tuple[Dict[str, Any], float]]:
+        valid_models = [(model, score) for model, score in model_scores if score > 0.01]
+        
+        if not valid_models:
+            valid_models = random.sample(model_scores, min(3, len(model_scores)))
+        
+        return valid_models[:top_k]
+    
+    def generate_responses_from_models(self, user_input: str, top_models: List[Tuple[Dict[str, Any], float]]) -> List[Tuple[str, float]]:
+        responses = []
+        input_pattern = self.pattern_extractor.create_pattern(user_input)
+        
+        for model, model_score in top_models:
+            model_responses = []
+            best_similarity = 0.0
+            
+            patterns = model.get('patterns', [])
+            model_responses_list = model.get('responses', [])
+            
+            if not patterns or not model_responses_list:
+                continue
+            
+            best_matches = []
+            for i, pattern in enumerate(patterns):
+                if i < len(model_responses_list):
+                    sim = self.pattern_extractor.calculate_pattern_similarity(input_pattern, pattern)
+                    if sim > 0.1:
+                        best_matches.append((model_responses_list[i], sim))
+            
+            best_matches.sort(key=lambda x: x[1], reverse=True)
+            
+            selected_responses = best_matches[:3] if best_matches else [(random.choice(model_responses_list), 0.1)]
+            
+            for response, pattern_sim in selected_responses:
+                weight = model_score * (0.7 + pattern_sim * 0.3)
+                responses.append((response, weight))
+        
+        return responses
+    
+    def combine_responses(self, responses: List[Tuple[str, float]]) -> str:
+        if not responses:
+            return "I'm not sure how to respond to that."
+        
+        filtered_responses = [(resp, weight) for resp, weight in responses if weight > 0.05]
+        if not filtered_responses:
+            filtered_responses = responses[:1]
+        
+        response_groups = defaultdict(list)
+        for response, weight in filtered_responses:
+            key = ' '.join(response.split()[:3]).lower()
+            response_groups[key].append((response, weight))
+        
+        best_responses = []
+        for group in response_groups.values():
+            best_resp, best_weight = max(group, key=lambda x: x[1])
+            best_responses.append((best_resp, best_weight))
+        
+        if len(best_responses) > 1:
+            total_weight = sum(weight for _, weight in best_responses)
+            if total_weight > 0:
+                normalized_weights = [weight / total_weight for _, weight in best_responses]
+                
+                rand_val = random.random()
+                cumsum = 0.0
+                for i, norm_weight in enumerate(normalized_weights):
+                    cumsum += norm_weight
+                    if rand_val <= cumsum:
+                        selected_response = best_responses[i][0]
+                        break
+                else:
+                    selected_response = best_responses[0][0]
+            else:
+                selected_response = best_responses[0][0]
+        else:
+            selected_response = best_responses[0][0]
+        
+        final_response = selected_response
+        
+        if not final_response.endswith('<eos>'):
+            final_response += ' <eos>'
+        
+        return final_response
+    
+    def generate_response(self, user_input: str) -> str:
+        if not user_input.strip():
+            return "Please say something! <eos>"
+        
+        model_scores = self.calculate_model_scores(user_input)
+        
+        if not model_scores:
+            return "I need to learn more before I can respond properly. <eos>"
+        
+        top_models = self.select_top_models(model_scores, top_k=5)
+        
+        responses = self.generate_responses_from_models(user_input, top_models)
+        
+        final_response = self.combine_responses(responses)
+        
+        self.context_window.append((user_input, final_response))
+        if len(self.context_window) > self.max_context_length:
+            self.context_window.pop(0)
+        
+        return final_response
+    
+    def get_model_statistics(self) -> Dict[str, Any]:
+        if not self.models:
+            return {"total_models": 0}
+        
+        total_patterns = sum(len(model.get('patterns', [])) for model in self.models)
+        total_responses = sum(len(model.get('responses', [])) for model in self.models)
+        avg_confidence = sum(model.get('confidence', 0) for model in self.models) / len(self.models)
+        total_training_samples = sum(model.get('training_samples', 0) for model in self.models)
+        
+        return {
+            "total_models": len(self.models),
+            "total_patterns": total_patterns,
+            "total_responses": total_responses,
+            "average_confidence": avg_confidence,
+            "total_training_samples": total_training_samples
+        }
+
+
+class AgGPTChat:
+    
+    def __init__(self, models_dir: str = "models"):
+        self.feather_manager = FeatherManager(models_dir)
+        self.response_generator = ResponseGenerator(self.feather_manager)
+        self.conversation_history = []
+    
+    def initialize(self) -> bool:
+        print("AgGPT-20 Scalable Feather Architecture Chat")
+        print("=" * 50)
+        
+        success = self.response_generator.load_models()
+        if success:
+            stats = self.response_generator.get_model_statistics()
+            print(f"Model Statistics:")
+            print(f"   Mini-models loaded: {stats['total_models']}")
+            print(f"   Total patterns: {stats['total_patterns']}")
+            print(f"   Total responses: {stats['total_responses']}")
+            print(f"   Average confidence: {stats['average_confidence']:.3f}")
+            print(f"   Training samples: {stats['total_training_samples']}")
+            print("=" * 50)
+            print("Chat initialized! Type 'quit' to exit.")
+            print("Large context window active - I'll remember our conversation!")
+            print()
+        
+        return success
+    
+    def chat_loop(self):
+        if not self.initialize():
+            return
+        
+        while True:
+            try:
+                user_input = input("You: ").strip()
+                
+                if not user_input:
+                    continue
+                
+                if user_input.lower() in ['quit', 'exit', 'bye', 'goodbye']:
+                    print("AgGPT: Goodbye! Thanks for chatting with me! <eos>")
+                    break
+                
+                if user_input.lower() in ['stats', 'statistics']:
+                    stats = self.response_generator.get_model_statistics()
+                    print("Current Statistics:")
+                    for key, value in stats.items():
+                        print(f"   {key}: {value}")
+                    continue
+                
+                if user_input.lower() in ['clear', 'reset']:
+                    self.response_generator.context_window = []
+                    print("Context cleared!")
+                    continue
+                
+                print("AgGPT: ", end="", flush=True)
+                response = self.response_generator.generate_response(user_input)
+                
+                display_response = response.replace(' <eos>', '').replace('<eos>', '')
+                print(display_response)
+                print()
+                
+                self.conversation_history.append({
+                    'user': user_input,
+                    'assistant': display_response
+                })
+                
+            except KeyboardInterrupt:
+                print("\n\nAgGPT: Chat interrupted. Goodbye!")
+                break
+            except Exception as e:
+                print(f"\nError: {e}")
+                print("Let me try again...")
+                continue
+    
+    def batch_test(self, test_inputs: List[str]):
+        if not self.initialize():
+            return
+        
+        print("Running batch test...")
+        print("=" * 50)
+        
+        for i, test_input in enumerate(test_inputs, 1):
+            print(f"Test {i}: {test_input}")
+            response = self.response_generator.generate_response(test_input)
+            display_response = response.replace(' <eos>', '').replace('<eos>', '')
+            print(f"Response: {display_response}")
+            print("-" * 30)
+
+
+def main():
+    chat = AgGPTChat()
+    
+    import sys
+    if len(sys.argv) > 1 and sys.argv[1] == "test":
+        test_inputs = [
+            "hi",
+            "hello there",
+            "how are you?",
+            "what's your favorite color?",
+            "tell me a joke",
+            "thank you",
+            "goodbye"
+        ]
+        chat.batch_test(test_inputs)
+    else:
+        chat.chat_loop()
+
+
+if __name__ == "__main__":
+    main()

feather.py

@@ -0,0 +1,235 @@
+import pandas as pd
+import os
+from typing import Dict, List, Any, Optional
+import math
+
+
+class FeatherManager:
+    
+    def __init__(self, models_dir: str = "models"):
+        self.models_dir = models_dir
+        os.makedirs(models_dir, exist_ok=True)
+    
+    def save_mini_model(self, model_data: Dict[str, Any], model_id: int, filename: str = None) -> str:
+        """Save model with custom filename or default naming"""
+        if filename is None:
+            filename = f"AgGPT_Expert_{model_id:04d}.feather"
+        
+        filepath = os.path.join(self.models_dir, filename)
+        
+        patterns = model_data.get('patterns', [])
+        responses = model_data.get('responses', [])
+        
+        if not patterns or not responses:
+            print(f"Warning: Model {model_id} has empty patterns or responses")
+            patterns = patterns or ['hello']
+            responses = responses or ['Hello!']
+        
+        df_data = {
+            'patterns': [str(pattern) for pattern in patterns],
+            'responses': [str(response) for response in responses],
+            'weights': model_data.get('weights', [1.0] * len(patterns)),
+            'confidence': [model_data.get('confidence', 0.5)] * len(patterns),
+            'grammar_rules': [str(rule) for rule in model_data.get('grammar_rules', [])] or ['none'],
+            'keywords': [' '.join(model_data.get('keywords', []))] * len(patterns),
+            'training_samples': [model_data.get('training_samples', 0)] * len(patterns)
+        }
+        
+        max_len = max(len(v) if isinstance(v, list) else 1 for v in df_data.values())
+        
+        for key, value in df_data.items():
+            if isinstance(value, list):
+                while len(value) < max_len:
+                    value.append(value[-1] if value else '')
+        
+        df = pd.DataFrame(df_data)
+        df.to_feather(filepath)
+        
+        print(f"Saved mini-model: {filename}")
+        return filepath
+    
+    def load_feather_file(self, filepath: str) -> Optional[Dict[str, Any]]:
+        """Load any .feather file and convert it to model format"""
+        if not os.path.exists(filepath):
+            return None
+        
+        try:
+            df = pd.read_feather(filepath)
+            
+            # Try to extract model ID from filename
+            filename = os.path.basename(filepath)
+            model_id = 0
+            
+            if filename.startswith("AgGPT_Expert_"):
+                try:
+                    model_id = int(filename.split("_")[2].split(".")[0])
+                except (ValueError, IndexError):
+                    model_id = hash(filename) % 10000
+            elif filename.startswith("target_AgGPT_"):
+                try:
+                    model_id = int(filename.split("_")[2].split(".")[0])
+                except (ValueError, IndexError):
+                    model_id = hash(filename) % 10000
+            else:
+                model_id = hash(filename) % 10000
+            
+            model_data = {
+                'patterns': [p for p in df['patterns'].tolist() if p],
+                'responses': [r for r in df['responses'].tolist() if r],
+                'weights': df['weights'].tolist() if 'weights' in df.columns else [1.0] * len(df),
+                'confidence': df['confidence'].iloc[0] if len(df) > 0 and 'confidence' in df.columns else 0.5,
+                'grammar_rules': [rule for rule in df['grammar_rules'].tolist() if rule] if 'grammar_rules' in df.columns else [],
+                'keywords': df['keywords'].iloc[0].split() if len(df) > 0 and 'keywords' in df.columns and df['keywords'].iloc[0] else [],
+                'training_samples': df['training_samples'].iloc[0] if len(df) > 0 and 'training_samples' in df.columns else 0,
+                'model_id': model_id,
+                'filename': filename
+            }
+            
+            return model_data
+            
+        except Exception as e:
+            print(f"Error loading feather file {filepath}: {e}")
+            return None
+    def load_mini_model(self, model_id: int) -> Optional[Dict[str, Any]]:
+        filename = f"AgGPT_Expert_{model_id:04d}.feather"
+        filepath = os.path.join(self.models_dir, filename)
+        return self.load_feather_file(filepath)
+    
+    def load_all_models(self) -> List[Dict[str, Any]]:
+        """Load ALL .feather files from the models directory"""
+        models = []
+        
+        if not os.path.exists(self.models_dir):
+            return models
+        
+        print(f"Scanning {self.models_dir} for .feather files...")
+        
+        for filename in sorted(os.listdir(self.models_dir)):
+            if filename.endswith(".feather"):
+                filepath = os.path.join(self.models_dir, filename)
+                model = self.load_feather_file(filepath)
+                if model:
+                    models.append(model)
+                    print(f"  ✅ Loaded: {filename}")
+                else:
+                    print(f"  ❌ Failed to load: {filename}")
+        
+        print(f"Total models loaded: {len(models)}")
+        return models
+    
+    def get_model_count(self) -> int:
+        """Count ALL .feather files in the models directory"""
+        if not os.path.exists(self.models_dir):
+            return 0
+        
+        count = 0
+        for filename in os.listdir(self.models_dir):
+            if filename.endswith(".feather"):
+                count += 1
+        
+        return count
+    
+    def get_next_model_id(self) -> int:
+        """Get next available model ID for AgGPT_Expert_ files"""
+        if not os.path.exists(self.models_dir):
+            return 1
+        
+        max_id = 0
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                try:
+                    model_id = int(filename.split("_")[2].split(".")[0])
+                    max_id = max(max_id, model_id)
+                except (ValueError, IndexError):
+                    continue
+            elif filename.endswith(".feather"):
+                # For non-standard named files, use a high number to avoid conflicts
+                try:
+                    # Extract numbers from any part of the filename
+                    import re
+                    numbers = re.findall(r'\d+', filename)
+                    if numbers:
+                        file_id = int(numbers[-1])  # Use the last number found
+                        max_id = max(max_id, file_id)
+                except (ValueError, IndexError):
+                    continue
+        
+        return max_id + 1
+    
+    def delete_model(self, model_id: int) -> bool:
+        filename = f"AgGPT_Expert_{model_id:04d}.feather"
+        filepath = os.path.join(self.models_dir, filename)
+        
+        if os.path.exists(filepath):
+            try:
+                os.remove(filepath)
+                print(f"Deleted model: {filename}")
+                return True
+            except Exception as e:
+                print(f"Error deleting model {model_id}: {e}")
+                return False
+        
+        return False
+    
+    def clear_all_models(self) -> int:
+        if not os.path.exists(self.models_dir):
+            return 0
+        
+        deleted_count = 0
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                try:
+                    os.remove(os.path.join(self.models_dir, filename))
+                    deleted_count += 1
+                except Exception as e:
+                    print(f"Error deleting {filename}: {e}")
+        
+        print(f"Deleted {deleted_count} model files")
+        return deleted_count
+
+
+def similarity_score(text1: str, text2: str) -> float:
+    if not text1 or not text2:
+        return 0.0
+    
+    words1 = set(text1.lower().split())
+    words2 = set(text2.lower().split())
+    
+    if not words1 or not words2:
+        return 0.0
+    
+    intersection = len(words1.intersection(words2))
+    union = len(words1.union(words2))
+    
+    return intersection / union if union > 0 else 0.0
+
+
+def calculate_confidence_score(patterns: List[str], responses: List[str]) -> float:
+    if not patterns or not responses or len(patterns) != len(responses):
+        return 0.1
+    
+    base_confidence = min(0.9, len(patterns) / 10.0)
+    
+    return max(0.1, min(1.0, base_confidence))
+
+
+if __name__ == "__main__":
+    manager = FeatherManager()
+    
+    test_model = {
+        'patterns': ['hello', 'hi', 'hey'],
+        'responses': ['Hello! How can I help you?', 'Hi there!', 'Hey! What\'s up?'],
+        'weights': [1.0, 0.9, 0.8],
+        'confidence': 0.8,
+        'grammar_rules': ['capitalize_first_word', 'end_with_punctuation'],
+        'keywords': ['greeting', 'hello', 'hi'],
+        'training_samples': 150
+    }
+    
+    model_id = manager.get_next_model_id()
+    manager.save_mini_model(test_model, model_id)
+    loaded_model = manager.load_mini_model(model_id)
+    
+    print(f"Original model: {test_model}")
+    print(f"Loaded model: {loaded_model}")
+    print(f"Models count: {manager.get_model_count()}")

target_train.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""
+AgGPT-20 Target Trainer
+Simple single-file training script that creates individual .feather models
+
+Usage:
+    python target_train.py [file_path]
+    
+If no file path is provided, it will use extra_corpora/example.txt by default.
+"""
+
+import os
+import glob
+from train import AgGPTTrainer, MiniModelTrainer
+from feather import FeatherManager
+
+def get_next_target_model_number(models_dir="models"):
+    """Find the next available target_AgGPT_X.feather number"""
+    if not os.path.exists(models_dir):
+        os.makedirs(models_dir)
+    
+    # Find all existing target_AgGPT_*.feather files
+    pattern = os.path.join(models_dir, "target_AgGPT_*.feather")
+    existing_files = glob.glob(pattern)
+    
+    if not existing_files:
+        return 1
+    
+    # Extract numbers from existing files
+    numbers = []
+    for file_path in existing_files:
+        filename = os.path.basename(file_path)
+        # Extract number from target_AgGPT_X.feather
+        try:
+            number_str = filename.split('_')[2].split('.')[0]
+            numbers.append(int(number_str))
+        except (IndexError, ValueError):
+            continue
+    
+    return max(numbers) + 1 if numbers else 1
+
+def train_target_model(file_path="extra_corpora/example.txt", models_dir="models"):
+    """Train a single target model from a text file"""
+    print("AgGPT-20 Target Trainer")
+    print("=" * 40)
+    
+    # Check if file exists
+    if not os.path.exists(file_path):
+        print(f"Error: Training file '{file_path}' not found!")
+        print(f"Please create the file or specify a different path.")
+        return False
+    
+    # Check file size
+    file_size = os.path.getsize(file_path)
+    if file_size == 0:
+        print(f"Error: Training file '{file_path}' is empty!")
+        return False
+    
+    print(f"Training file: {file_path}")
+    print(f"File size: {file_size} bytes")
+    
+    # Create models directory if it doesn't exist
+    if not os.path.exists(models_dir):
+        os.makedirs(models_dir)
+    
+    # Initialize trainer components
+    feather_manager = FeatherManager(models_dir)
+    trainer = AgGPTTrainer(models_dir)
+    mini_trainer = MiniModelTrainer(feather_manager)
+    
+    # Load training data
+    print(f"Loading training data from {file_path}...")
+    try:
+        training_pairs = trainer.load_training_data(file_path)
+    except Exception as e:
+        print(f"Error loading training data: {e}")
+        return False
+    
+    if not training_pairs:
+        print("No valid training pairs found in the file!")
+        print("Please check the file format:")
+        print("user: [your message]")
+        print("<pad>")
+        print("ai: [response]")
+        print("<eos>")
+        return False
+    
+    print(f"Found {len(training_pairs)} training pairs")
+    
+    # Train the mini-model
+    print("Training mini-model...")
+    try:
+        mini_model = mini_trainer.train_mini_model(training_pairs)
+    except Exception as e:
+        print(f"Error during training: {e}")
+        return False
+    
+    if not mini_model:
+        print("Failed to create mini-model!")
+        return False
+    
+    # Get next available model number
+    model_number = get_next_target_model_number(models_dir)
+    model_filename = f"target_AgGPT_{model_number}.feather"
+    
+    # Save the model with custom filename
+    print(f"Saving model as {model_filename}...")
+    try:
+        feather_manager.save_mini_model(mini_model, model_number, model_filename)
+        model_path = os.path.join(models_dir, model_filename)
+        print(f"✅ Successfully created: {model_path}")
+        
+    except Exception as e:
+        print(f"Error saving model: {e}")
+        return False
+    
+    # Display model statistics
+    print("\nModel Statistics:")
+    print(f"  Training samples: {mini_model.get('training_samples', 0)}")
+    print(f"  Patterns: {len(mini_model.get('patterns', []))}")
+    print(f"  Responses: {len(mini_model.get('responses', []))}")
+    print(f"  Confidence: {mini_model.get('confidence', 0.0):.3f}")
+    print(f"  Keywords: {len(mini_model.get('keywords', []))}")
+    
+    print(f"\n🎉 Target model training complete!")
+    print(f"The model will be automatically discovered by chat.py")
+    print("Run 'python chat.py' to test your new model!")
+    
+    return True
+
+def main():
+    import sys
+    
+    # Check command line arguments
+    if len(sys.argv) > 1:
+        file_path = sys.argv[1]
+    else:
+        file_path = "extra_corpora/example.txt"
+    
+    success = train_target_model(file_path)
+    
+    if not success:
+        print("\n❌ Training failed!")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

train.py

@@ -0,0 +1,371 @@
+import os
+import re
+import random
+import math
+import yaml
+from typing import List, Dict, Tuple, Set, Any
+from collections import defaultdict, Counter
+import pandas as pd
+from tqdm import tqdm
+
+from feather import FeatherManager, similarity_score, calculate_confidence_score
+
+class GrammarRules:
+    @staticmethod
+    def apply_all_rules(text: str) -> str:
+        if not text:
+            return text
+        
+        return text.strip()
+
+class PatternExtractor:
+    
+    def __init__(self):
+        pass
+    
+    def extract_keywords(self, text: str) -> List[str]:
+        if not text:
+            return []
+        
+        full_text_normalized = re.sub(r'\s+', ' ', text.strip().lower())
+        words = re.findall(r'\b[a-zA-Z]+\b', full_text_normalized)
+        
+        result = [full_text_normalized]
+        result.extend(words)
+        
+        return list(set(result))
+    
+    def create_pattern(self, user_input: str) -> str:
+        if not user_input:
+            return ""
+        
+        pattern = re.sub(r'\s+', ' ', user_input.strip().lower())
+        
+        pattern = f" {pattern} "
+        
+        return pattern
+    
+    def calculate_pattern_similarity(self, pattern1: str, pattern2: str) -> float:
+        return similarity_score(pattern1.strip(), pattern2.strip())
+
+
+class MiniModelTrainer:
+    
+    def __init__(self, feather_manager: FeatherManager):
+        self.feather_manager = feather_manager
+        self.pattern_extractor = PatternExtractor()
+        self.grammar_rules = GrammarRules()
+    
+    def train_mini_model(self, training_pairs: List[Tuple[str, str]], confidence_threshold: float = 0.1) -> Dict[str, Any]:
+        if not training_pairs or len(training_pairs) < 2:
+            return None
+
+        keyword_patterns = []
+        responses = []
+        pattern_confidences = []
+        all_keywords = []
+        
+        for user_input, ai_response in training_pairs:
+            processed_response = ai_response.strip()
+            
+            # Get both the pattern and keywords
+            pattern = self.pattern_extractor.create_pattern(user_input)
+            keywords = self.pattern_extractor.extract_keywords(user_input)
+            all_keywords.extend(keywords)
+            
+            # Add the main pattern
+            keyword_patterns.append(pattern)
+            responses.append(processed_response)
+            individual_confidence = min(0.9, len(training_pairs) / 20.0)
+            pattern_confidences.append(individual_confidence)
+        
+        if not keyword_patterns:
+            return None
+        
+        base_confidence = min(0.9, len(training_pairs) / 20.0)
+        keyword_counter = Counter(all_keywords)
+        top_keywords = [word for word, count in keyword_counter.most_common(10)]
+        
+        mini_model = {
+            'patterns': keyword_patterns,
+            'responses': responses,
+            'pattern_confidences': pattern_confidences,
+            'confidence': base_confidence,
+            'grammar_rules': [],
+            'keywords': top_keywords,
+            'training_samples': len(training_pairs)
+        }
+        
+        return mini_model
+
+
+class AgGPTTrainer:
+    
+    def __init__(self, models_dir: str = "models"):
+        self.feather_manager = FeatherManager(models_dir)
+        self.mini_trainer = MiniModelTrainer(self.feather_manager)
+        self.target_size_mb = 5
+        self.estimated_size_per_pair = 1000
+        self.chunk_size = (self.target_size_mb * 1024 * 1024) // self.estimated_size_per_pair
+        self.readable_weights_dir = "readable_weights"
+        os.makedirs(self.readable_weights_dir, exist_ok=True)
+    
+    def save_model_as_yaml(self, model_data: Dict[str, Any], model_id: int):
+        try:
+            filename = f"AgGPT_Model_{model_id:04d}.yaml"
+            filepath = os.path.join(self.readable_weights_dir, filename)
+            
+            print(f"Creating YAML data for model {model_id}...")
+            yaml_data = {
+                'model_info': {
+                    'model_id': model_id,
+                    'confidence': model_data.get('confidence', 0.5),
+                    'training_samples': model_data.get('training_samples', 0),
+                    'keywords': model_data.get('keywords', [])
+                },
+                'patterns_and_responses': []
+            }
+            
+            patterns = model_data.get('patterns', [])
+            responses = model_data.get('responses', [])
+            weights = model_data.get('weights', [])
+            
+            print(f"Processing {len(patterns)} patterns...")
+            for i in range(len(patterns)):
+                entry = {
+                    'pattern': patterns[i] if i < len(patterns) else '',
+                    'response': responses[i] if i < len(responses) else '',
+                    'weight': weights[i] if i < len(weights) else 1.0
+                }
+                yaml_data['patterns_and_responses'].append(entry)
+            
+            print(f"Writing YAML to {filepath}...")
+            with open(filepath, 'w', encoding='utf-8') as f:
+                yaml.dump(yaml_data, f, default_flow_style=False, allow_unicode=True, indent=2)
+            
+            print(f"Saved readable model: {filename}")
+        except Exception as e:
+            print(f"Error in save_model_as_yaml: {e}")
+            import traceback
+            traceback.print_exc()
+    
+    def load_training_data(self, file_path: str) -> List[Tuple[str, str]]:
+        training_pairs = []
+        
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        
+        conversations = content.split('<eos>')
+        
+        print(f"Processing {len(conversations)} conversation chunks...")
+        
+        for conversation in tqdm(conversations, desc="Parsing conversations"):
+            conversation = conversation.strip()
+            if not conversation:
+                continue
+            
+            user_match = re.search(r'user:\s*(.*?)(?=\n<pad>|\nai:|$)', conversation, re.DOTALL)
+            ai_match = re.search(r'ai:\s*(.*?)$', conversation, re.DOTALL)
+            
+            if user_match and ai_match:
+                user_input = user_match.group(1).strip()
+                ai_response = ai_match.group(1).strip()
+                
+                user_input = re.sub(r'<pad>', '', user_input).strip()
+                ai_response = re.sub(r'<pad>', '', ai_response).strip()
+                
+                if user_input and ai_response and len(user_input) > 0 and len(ai_response) > 0:
+                    training_pairs.append((user_input, ai_response))
+        
+        print(f"Extracted {len(training_pairs)} training pairs")
+        return training_pairs
+    
+    def create_training_chunks(self, training_pairs: List[Tuple[str, str]]) -> List[List[Tuple[str, str]]]:
+        shuffled_pairs = training_pairs.copy()
+        random.shuffle(shuffled_pairs)
+        
+        chunks = []
+        total_pairs = len(shuffled_pairs)
+        
+        for i in range(0, total_pairs, self.chunk_size):
+            chunk = shuffled_pairs[i:i + self.chunk_size]
+            if len(chunk) >= 5:
+                chunks.append(chunk)
+        
+        print(f"Created {len(chunks)} training chunks (target: {self.target_size_mb}MB each)")
+        return chunks
+    
+    def train_multiple_corpora(self, training_files: List[str] = None):
+        """Train on multiple corpora files sequentially"""
+        if training_files is None:
+            # Automatically find all .txt files in training_corpora directory
+            training_corpora_dir = "training_corpora"
+            if os.path.exists(training_corpora_dir):
+                training_files = []
+                for filename in sorted(os.listdir(training_corpora_dir)):
+                    if filename.endswith('.txt'):
+                        file_path = os.path.join(training_corpora_dir, filename)
+                        training_files.append(file_path)
+                print(f"Auto-discovered {len(training_files)} training files in {training_corpora_dir}/")
+            else:
+                # Fallback to old default for backward compatibility
+                training_files = ["training_data/corpora.txt", "training_data/corpora2.txt"]
+                print("Using fallback training files (training_corpora/ not found)")
+        
+        print("Starting AgGPT-20 Multi-Corpora Training with Scalable Feather Architecture")
+        print("=" * 70)
+        
+        cleared_count = self.feather_manager.clear_all_models()
+        if cleared_count > 0:
+            print(f"Cleared {cleared_count} existing models")
+        
+        all_trained_models = []
+        total_model_id = 1
+        
+        for file_idx, training_file in enumerate(training_files, 1):
+            print(f"\n--- Training on file {file_idx}/{len(training_files)}: {training_file} ---")
+            
+            if not os.path.exists(training_file):
+                print(f"Warning: Training file {training_file} does not exist. Skipping...")
+                continue
+            
+            if os.path.getsize(training_file) == 0:
+                print(f"Warning: Training file {training_file} is empty. Skipping...")
+                continue
+            
+            print(f"Loading training data from {training_file}...")
+            training_pairs = self.load_training_data(training_file)
+            
+            if not training_pairs:
+                print(f"No training data found in {training_file}. Skipping...")
+                continue
+            
+            print(f"Creating training chunks for {training_file}...")
+            training_chunks = self.create_training_chunks(training_pairs)
+            
+            print(f"Training mini-models from {training_file}...")
+            file_trained_models = []
+            
+            progress_bar = tqdm(training_chunks, desc=f"Training from {os.path.basename(training_file)}")
+            for chunk_idx, chunk in enumerate(progress_bar):
+                print(f"\nProcessing chunk {chunk_idx + 1}/{len(training_chunks)}")
+                mini_model = self.mini_trainer.train_mini_model(chunk)
+                
+                if mini_model:
+                    file_trained_models.append(mini_model)
+                    all_trained_models.append(mini_model)
+                    print(f"Saving model {total_model_id}...")
+                    self.feather_manager.save_mini_model(mini_model, total_model_id)
+                    
+                    if total_model_id == 1:
+                        print("Saving first model as YAML...")
+                        try:
+                            self.save_model_as_yaml(mini_model, total_model_id)
+                            print("YAML saved successfully")
+                        except Exception as e:
+                            print(f"Error saving YAML: {e}")
+                    
+                    total_model_id += 1
+                    print(f"Model {total_model_id - 1} completed")
+                    
+                    try:
+                        progress_bar.set_postfix({
+                            'File Models': len(file_trained_models),
+                            'Total Models': len(all_trained_models),
+                            'Confidence': f"{mini_model['confidence']:.3f}"
+                        })
+                    except Exception as e:
+                        print(f"Error updating progress bar: {e}")
+            
+            print(f"Completed training on {training_file}: {len(file_trained_models)} mini-models created")
+            print(f"Total models so far: {len(all_trained_models)}")
+        
+        print(f"\n--- Multi-Corpora Training Complete ---")
+        final_count = self.feather_manager.get_model_count()
+        print(f"Final model count: {final_count}")
+        print(f"Trained on {len([f for f in training_files if os.path.exists(f) and os.path.getsize(f) > 0])} corpora files")
+        print("=" * 70)
+    
+    def train(self, training_file: str = "training_data/corpora.txt"):
+        print("Starting AgGPT-20 Training with Scalable Feather Architecture")
+        print("=" * 60)
+        
+        cleared_count = self.feather_manager.clear_all_models()
+        if cleared_count > 0:
+            print(f"Cleared {cleared_count} existing models")
+        
+        print("Loading training data...")
+        training_pairs = self.load_training_data(training_file)
+        
+        if not training_pairs:
+            print("No training data found!")
+            return
+        
+        print("Creating training chunks...")
+        training_chunks = self.create_training_chunks(training_pairs)
+        
+        print("Training mini-models...")
+        trained_models = []
+        model_id = 1
+        
+        progress_bar = tqdm(training_chunks, desc="Training mini-models")
+        for chunk in progress_bar:
+            mini_model = self.mini_trainer.train_mini_model(chunk)
+            
+            if mini_model:
+                trained_models.append(mini_model)
+                self.feather_manager.save_mini_model(mini_model, model_id)
+                
+                if model_id == 1:
+                    self.save_model_as_yaml(mini_model, model_id)
+                
+                model_id += 1
+                
+                progress_bar.set_postfix({
+                    'Models': len(trained_models),
+                    'Confidence': f"{mini_model['confidence']:.3f}"
+                })
+        
+        print(f"Trained {len(trained_models)} mini-models")
+        
+        final_count = self.feather_manager.get_model_count()
+        print(f"Training complete! Final model count: {final_count}")
+        print("=" * 60)
+
+
+def main():
+    print("AgGPT-20 Scalable Feather Architecture Trainer")
+    print("=" * 50)
+    
+    # Automatically find all .txt files in training_corpora directory
+    training_corpora_dir = "training_corpora"
+    training_files = []
+    
+    if os.path.exists(training_corpora_dir):
+        print(f"Scanning {training_corpora_dir} directory for training files...")
+        for filename in sorted(os.listdir(training_corpora_dir)):
+            if filename.endswith('.txt'):
+                file_path = os.path.join(training_corpora_dir, filename)
+                training_files.append(file_path)
+        
+        print(f"Found {len(training_files)} training files:")
+        for file_path in training_files:
+            print(f"  - {file_path}")
+    else:
+        print(f"Warning: {training_corpora_dir} directory not found!")
+        training_files = None
+    
+    trainer = AgGPTTrainer()
+    
+    try:
+        trainer.train_multiple_corpora(training_files=training_files)
+    except KeyboardInterrupt:
+        print("\nTraining interrupted by user")
+    except Exception as e:
+        print(f"Training failed: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()