Upload 4 files

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 banner.png filter=lfs diff=lfs merge=lfs -text
+training_data/corpora.txt filter=lfs diff=lfs merge=lfs -text

chat.py

@@ -0,0 +1,321 @@
+import os
+import re
+import random
+from typing import List, Dict, Tuple, Any
+from collections import defaultdict
+import math
+
+from feather import FeatherManager, similarity_score
+from train import GrammarRules, PatternExtractor
+
+
+class ResponseGenerator:
+    
+    def __init__(self, feather_manager: FeatherManager):
+        self.feather_manager = feather_manager
+        self.pattern_extractor = PatternExtractor()
+        self.grammar_rules = GrammarRules()
+        self.models = []
+        self.context_window = []
+        self.max_context_length = 10
+        
+    def load_models(self):
+        print("Loading mini-models...")
+        self.models = self.feather_manager.load_all_models()
+        print(f"Loaded {len(self.models)} mini-models")
+        
+        if not self.models:
+            print("No trained models found! Please run train.py first.")
+            return False
+        
+        return True
+    
+    def calculate_model_scores(self, user_input: str) -> List[Tuple[Dict[str, Any], float]]:
+        if not self.models:
+            return []
+        
+        input_pattern = self.pattern_extractor.create_pattern(user_input)
+        input_keywords = set(self.pattern_extractor.extract_keywords(user_input))
+        
+        model_scores = []
+        
+        for model in self.models:
+            score = 0.0
+            pattern_matches = 0
+            keyword_matches = 0
+            
+            for pattern in model.get('patterns', []):
+                pattern_sim = self.pattern_extractor.calculate_pattern_similarity(input_pattern, pattern)
+                score += pattern_sim
+                if pattern_sim > 0.3:
+                    pattern_matches += 1
+            
+            model_keywords = set(model.get('keywords', []))
+            if model_keywords and input_keywords:
+                keyword_overlap = len(input_keywords.intersection(model_keywords))
+                keyword_total = len(input_keywords.union(model_keywords))
+                keyword_score = keyword_overlap / keyword_total if keyword_total > 0 else 0
+                score += keyword_score * 2
+                keyword_matches = keyword_overlap
+            
+            confidence = model.get('confidence', 0.5)
+            score *= confidence
+            
+            training_samples = model.get('training_samples', 1)
+            training_bonus = min(0.2, training_samples / 100)
+            score += training_bonus
+            
+            context_bonus = self._calculate_context_bonus(user_input, model)
+            score += context_bonus
+            
+            model_scores.append((model, score))
+        
+        model_scores.sort(key=lambda x: x[1], reverse=True)
+        
+        return model_scores
+    
+    def _calculate_context_bonus(self, user_input: str, model: Dict[str, Any]) -> float:
+        if not self.context_window:
+            return 0.0
+        
+        context_bonus = 0.0
+        
+        for prev_input, prev_response in self.context_window[-3:]:
+            for pattern in model.get('patterns', [])[:5]:
+                pattern_sim = similarity_score(prev_input, pattern.strip())
+                context_bonus += pattern_sim * 0.1
+        
+        return min(context_bonus, 0.3)
+    
+    def select_top_models(self, model_scores: List[Tuple[Dict[str, Any], float]], top_k: int = 5) -> List[Tuple[Dict[str, Any], float]]:
+        valid_models = [(model, score) for model, score in model_scores if score > 0.01]
+        
+        if not valid_models:
+            valid_models = random.sample(model_scores, min(3, len(model_scores)))
+        
+        return valid_models[:top_k]
+    
+    def generate_responses_from_models(self, user_input: str, top_models: List[Tuple[Dict[str, Any], float]]) -> List[Tuple[str, float]]:
+        responses = []
+        input_pattern = self.pattern_extractor.create_pattern(user_input)
+        
+        for model, model_score in top_models:
+            model_responses = []
+            best_similarity = 0.0
+            
+            patterns = model.get('patterns', [])
+            model_responses_list = model.get('responses', [])
+            
+            if not patterns or not model_responses_list:
+                continue
+            
+            best_matches = []
+            for i, pattern in enumerate(patterns):
+                if i < len(model_responses_list):
+                    sim = self.pattern_extractor.calculate_pattern_similarity(input_pattern, pattern)
+                    if sim > 0.1:
+                        best_matches.append((model_responses_list[i], sim))
+            
+            best_matches.sort(key=lambda x: x[1], reverse=True)
+            
+            selected_responses = best_matches[:3] if best_matches else [(random.choice(model_responses_list), 0.1)]
+            
+            for response, pattern_sim in selected_responses:
+                weight = model_score * (0.7 + pattern_sim * 0.3)
+                responses.append((response, weight))
+        
+        return responses
+    
+    def combine_responses(self, responses: List[Tuple[str, float]]) -> str:
+        if not responses:
+            return "I'm not sure how to respond to that."
+        
+        filtered_responses = [(resp, weight) for resp, weight in responses if weight > 0.05]
+        if not filtered_responses:
+            filtered_responses = responses[:1]
+        
+        response_groups = defaultdict(list)
+        for response, weight in filtered_responses:
+            key = ' '.join(response.split()[:3]).lower()
+            response_groups[key].append((response, weight))
+        
+        best_responses = []
+        for group in response_groups.values():
+            best_resp, best_weight = max(group, key=lambda x: x[1])
+            best_responses.append((best_resp, best_weight))
+        
+        if len(best_responses) > 1:
+            total_weight = sum(weight for _, weight in best_responses)
+            if total_weight > 0:
+                normalized_weights = [weight / total_weight for _, weight in best_responses]
+                
+                rand_val = random.random()
+                cumsum = 0.0
+                for i, norm_weight in enumerate(normalized_weights):
+                    cumsum += norm_weight
+                    if rand_val <= cumsum:
+                        selected_response = best_responses[i][0]
+                        break
+                else:
+                    selected_response = best_responses[0][0]
+            else:
+                selected_response = best_responses[0][0]
+        else:
+            selected_response = best_responses[0][0]
+        
+        final_response = selected_response
+        
+        if not final_response.endswith('<eos>'):
+            final_response += ' <eos>'
+        
+        return final_response
+    
+    def generate_response(self, user_input: str) -> str:
+        if not user_input.strip():
+            return "Please say something! <eos>"
+        
+        model_scores = self.calculate_model_scores(user_input)
+        
+        if not model_scores:
+            return "I need to learn more before I can respond properly. <eos>"
+        
+        top_models = self.select_top_models(model_scores, top_k=5)
+        
+        responses = self.generate_responses_from_models(user_input, top_models)
+        
+        final_response = self.combine_responses(responses)
+        
+        self.context_window.append((user_input, final_response))
+        if len(self.context_window) > self.max_context_length:
+            self.context_window.pop(0)
+        
+        return final_response
+    
+    def get_model_statistics(self) -> Dict[str, Any]:
+        if not self.models:
+            return {"total_models": 0}
+        
+        total_patterns = sum(len(model.get('patterns', [])) for model in self.models)
+        total_responses = sum(len(model.get('responses', [])) for model in self.models)
+        avg_confidence = sum(model.get('confidence', 0) for model in self.models) / len(self.models)
+        total_training_samples = sum(model.get('training_samples', 0) for model in self.models)
+        
+        return {
+            "total_models": len(self.models),
+            "total_patterns": total_patterns,
+            "total_responses": total_responses,
+            "average_confidence": avg_confidence,
+            "total_training_samples": total_training_samples
+        }
+
+
+class AgGPTChat:
+    
+    def __init__(self, models_dir: str = "models"):
+        self.feather_manager = FeatherManager(models_dir)
+        self.response_generator = ResponseGenerator(self.feather_manager)
+        self.conversation_history = []
+    
+    def initialize(self) -> bool:
+        print("AgGPT-17 Scalable Feather Architecture Chat")
+        print("=" * 50)
+        
+        success = self.response_generator.load_models()
+        if success:
+            stats = self.response_generator.get_model_statistics()
+            print(f"Model Statistics:")
+            print(f"   Mini-models loaded: {stats['total_models']}")
+            print(f"   Total patterns: {stats['total_patterns']}")
+            print(f"   Total responses: {stats['total_responses']}")
+            print(f"   Average confidence: {stats['average_confidence']:.3f}")
+            print(f"   Training samples: {stats['total_training_samples']}")
+            print("=" * 50)
+            print("Chat initialized! Type 'quit' to exit.")
+            print("Large context window active - I'll remember our conversation!")
+            print()
+        
+        return success
+    
+    def chat_loop(self):
+        if not self.initialize():
+            return
+        
+        while True:
+            try:
+                user_input = input("You: ").strip()
+                
+                if not user_input:
+                    continue
+                
+                if user_input.lower() in ['quit', 'exit', 'bye', 'goodbye']:
+                    print("AgGPT: Goodbye! Thanks for chatting with me! <eos>")
+                    break
+                
+                if user_input.lower() in ['stats', 'statistics']:
+                    stats = self.response_generator.get_model_statistics()
+                    print("Current Statistics:")
+                    for key, value in stats.items():
+                        print(f"   {key}: {value}")
+                    continue
+                
+                if user_input.lower() in ['clear', 'reset']:
+                    self.response_generator.context_window = []
+                    print("Context cleared!")
+                    continue
+                
+                print("AgGPT: ", end="", flush=True)
+                response = self.response_generator.generate_response(user_input)
+                
+                display_response = response.replace(' <eos>', '').replace('<eos>', '')
+                print(display_response)
+                print()
+                
+                self.conversation_history.append({
+                    'user': user_input,
+                    'assistant': display_response
+                })
+                
+            except KeyboardInterrupt:
+                print("\n\nAgGPT: Chat interrupted. Goodbye!")
+                break
+            except Exception as e:
+                print(f"\nError: {e}")
+                print("Let me try again...")
+                continue
+    
+    def batch_test(self, test_inputs: List[str]):
+        if not self.initialize():
+            return
+        
+        print("Running batch test...")
+        print("=" * 50)
+        
+        for i, test_input in enumerate(test_inputs, 1):
+            print(f"Test {i}: {test_input}")
+            response = self.response_generator.generate_response(test_input)
+            display_response = response.replace(' <eos>', '').replace('<eos>', '')
+            print(f"Response: {display_response}")
+            print("-" * 30)
+
+
+def main():
+    chat = AgGPTChat()
+    
+    import sys
+    if len(sys.argv) > 1 and sys.argv[1] == "test":
+        test_inputs = [
+            "hi",
+            "hello there",
+            "how are you?",
+            "what's your favorite color?",
+            "tell me a joke",
+            "thank you",
+            "goodbye"
+        ]
+        chat.batch_test(test_inputs)
+    else:
+        chat.chat_loop()
+
+
+if __name__ == "__main__":
+    main()

feather.py

@@ -0,0 +1,196 @@
+import pandas as pd
+import os
+from typing import Dict, List, Any, Optional
+import math
+
+
+class FeatherManager:
+    
+    def __init__(self, models_dir: str = "models"):
+        self.models_dir = models_dir
+        os.makedirs(models_dir, exist_ok=True)
+    
+    def save_mini_model(self, model_data: Dict[str, Any], model_id: int) -> str:
+        filename = f"AgGPT_Expert_{model_id:04d}.feather"
+        filepath = os.path.join(self.models_dir, filename)
+        
+        patterns = model_data.get('patterns', [])
+        responses = model_data.get('responses', [])
+        
+        if not patterns or not responses:
+            print(f"Warning: Model {model_id} has empty patterns or responses")
+            patterns = patterns or ['hello']
+            responses = responses or ['Hello!']
+        
+        df_data = {
+            'patterns': [str(pattern) for pattern in patterns],
+            'responses': [str(response) for response in responses],
+            'weights': model_data.get('weights', [1.0] * len(patterns)),
+            'confidence': [model_data.get('confidence', 0.5)] * len(patterns),
+            'grammar_rules': [str(rule) for rule in model_data.get('grammar_rules', [])] or ['none'],
+            'keywords': [' '.join(model_data.get('keywords', []))] * len(patterns),
+            'training_samples': [model_data.get('training_samples', 0)] * len(patterns)
+        }
+        
+        max_len = max(len(v) if isinstance(v, list) else 1 for v in df_data.values())
+        
+        for key, value in df_data.items():
+            if isinstance(value, list):
+                while len(value) < max_len:
+                    value.append(value[-1] if value else '')
+        
+        df = pd.DataFrame(df_data)
+        df.to_feather(filepath)
+        
+        print(f"Saved mini-model: {filename}")
+        return filepath
+    
+    def load_mini_model(self, model_id: int) -> Optional[Dict[str, Any]]:
+        filename = f"AgGPT_Expert_{model_id:04d}.feather"
+        filepath = os.path.join(self.models_dir, filename)
+        
+        if not os.path.exists(filepath):
+            return None
+        
+        try:
+            df = pd.read_feather(filepath)
+            
+            model_data = {
+                'patterns': [p for p in df['patterns'].tolist() if p],
+                'responses': [r for r in df['responses'].tolist() if r],
+                'weights': df['weights'].tolist(),
+                'confidence': df['confidence'].iloc[0] if len(df) > 0 else 0.5,
+                'grammar_rules': [rule for rule in df['grammar_rules'].tolist() if rule],
+                'keywords': df['keywords'].iloc[0].split() if len(df) > 0 and df['keywords'].iloc[0] else [],
+                'training_samples': df['training_samples'].iloc[0] if len(df) > 0 else 0,
+                'model_id': model_id
+            }
+            
+            return model_data
+            
+        except Exception as e:
+            print(f"Error loading model {model_id}: {e}")
+            return None
+    
+    def load_all_models(self) -> List[Dict[str, Any]]:
+        models = []
+        
+        if not os.path.exists(self.models_dir):
+            return models
+        
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                try:
+                    model_id = int(filename.split("_")[2].split(".")[0])
+                    model = self.load_mini_model(model_id)
+                    if model:
+                        models.append(model)
+                except (ValueError, IndexError):
+                    print(f"Warning: Invalid model filename format: {filename}")
+                    continue
+        
+        return models
+    
+    def get_model_count(self) -> int:
+        if not os.path.exists(self.models_dir):
+            return 0
+        
+        count = 0
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                count += 1
+        
+        return count
+    
+    def get_next_model_id(self) -> int:
+        if not os.path.exists(self.models_dir):
+            return 1
+        
+        max_id = 0
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                try:
+                    model_id = int(filename.split("_")[2].split(".")[0])
+                    max_id = max(max_id, model_id)
+                except (ValueError, IndexError):
+                    continue
+        
+        return max_id + 1
+    
+    def delete_model(self, model_id: int) -> bool:
+        filename = f"AgGPT_Expert_{model_id:04d}.feather"
+        filepath = os.path.join(self.models_dir, filename)
+        
+        if os.path.exists(filepath):
+            try:
+                os.remove(filepath)
+                print(f"Deleted model: {filename}")
+                return True
+            except Exception as e:
+                print(f"Error deleting model {model_id}: {e}")
+                return False
+        
+        return False
+    
+    def clear_all_models(self) -> int:
+        if not os.path.exists(self.models_dir):
+            return 0
+        
+        deleted_count = 0
+        for filename in os.listdir(self.models_dir):
+            if filename.startswith("AgGPT_Expert_") and filename.endswith(".feather"):
+                try:
+                    os.remove(os.path.join(self.models_dir, filename))
+                    deleted_count += 1
+                except Exception as e:
+                    print(f"Error deleting {filename}: {e}")
+        
+        print(f"Deleted {deleted_count} model files")
+        return deleted_count
+
+
+def similarity_score(text1: str, text2: str) -> float:
+    if not text1 or not text2:
+        return 0.0
+    
+    words1 = set(text1.lower().split())
+    words2 = set(text2.lower().split())
+    
+    if not words1 or not words2:
+        return 0.0
+    
+    intersection = len(words1.intersection(words2))
+    union = len(words1.union(words2))
+    
+    return intersection / union if union > 0 else 0.0
+
+
+def calculate_confidence_score(patterns: List[str], responses: List[str]) -> float:
+    if not patterns or not responses or len(patterns) != len(responses):
+        return 0.1
+    
+    base_confidence = min(0.9, len(patterns) / 10.0)
+    
+    return max(0.1, min(1.0, base_confidence))
+
+
+if __name__ == "__main__":
+    manager = FeatherManager()
+    
+    test_model = {
+        'patterns': ['hello', 'hi', 'hey'],
+        'responses': ['Hello! How can I help you?', 'Hi there!', 'Hey! What\'s up?'],
+        'weights': [1.0, 0.9, 0.8],
+        'confidence': 0.8,
+        'grammar_rules': ['capitalize_first_word', 'end_with_punctuation'],
+        'keywords': ['greeting', 'hello', 'hi'],
+        'training_samples': 150
+    }
+    
+    model_id = manager.get_next_model_id()
+    manager.save_mini_model(test_model, model_id)
+    loaded_model = manager.load_mini_model(model_id)
+    
+    print(f"Original model: {test_model}")
+    print(f"Loaded model: {loaded_model}")
+    print(f"Models count: {manager.get_model_count()}")

train.py

@@ -0,0 +1,298 @@
+import os
+import re
+import random
+import math
+from typing import List, Dict, Tuple, Set, Any
+from collections import defaultdict, Counter
+import pandas as pd
+from tqdm import tqdm
+
+from feather import FeatherManager, similarity_score, calculate_confidence_score
+
+class GrammarRules:
+    
+    @staticmethod
+    def apply_all_rules(text: str) -> str:
+        if not text:
+            return text
+        
+        return text.strip()
+
+class PatternExtractor:
+    
+    def __init__(self):
+        self.stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'can', 'shall'}
+    
+    def extract_keywords(self, text: str) -> List[str]:
+        if not text:
+            return []
+        
+        words = re.findall(r'\b[a-zA-Z]+\b', text.lower())
+        
+        keywords = [word for word in words if word not in self.stop_words and len(word) > 2]
+        
+        return list(set(keywords))
+    
+    def create_pattern(self, user_input: str) -> str:
+        if not user_input:
+            return ""
+        
+        pattern = re.sub(r'\s+', ' ', user_input.strip().lower())
+        
+        pattern = f" {pattern} "
+        
+        return pattern
+    
+    def calculate_pattern_similarity(self, pattern1: str, pattern2: str) -> float:
+        return similarity_score(pattern1.strip(), pattern2.strip())
+
+
+class MiniModelTrainer:
+    
+    def __init__(self, feather_manager: FeatherManager):
+        self.feather_manager = feather_manager
+        self.pattern_extractor = PatternExtractor()
+        self.grammar_rules = GrammarRules()
+    
+    def train_mini_model(self, training_pairs: List[Tuple[str, str]], confidence_threshold: float = 0.1) -> Dict[str, Any]:
+        if not training_pairs or len(training_pairs) < 2:
+            return None
+
+        patterns = []
+        responses = []
+        weights = []
+        all_keywords = []
+        
+        for user_input, ai_response in training_pairs:
+            processed_response = ai_response.strip()
+            
+            pattern = self.pattern_extractor.create_pattern(user_input)
+            
+            keywords = self.pattern_extractor.extract_keywords(user_input)
+            all_keywords.extend(keywords)
+            
+            patterns.append(pattern)
+            responses.append(processed_response)
+            weights.append(1.0)
+        
+        confidence = min(0.9, len(training_pairs) / 20.0)
+        
+        keyword_counter = Counter(all_keywords)
+        top_keywords = [word for word, count in keyword_counter.most_common(10)]
+        
+        mini_model = {
+            'patterns': patterns,
+            'responses': responses,
+            'weights': weights,
+            'confidence': confidence,
+            'grammar_rules': [],
+            'keywords': top_keywords,
+            'training_samples': len(training_pairs)
+        }
+        
+        return mini_model
+    
+    def should_merge_models(self, model1: Dict[str, Any], model2: Dict[str, Any], merge_threshold: float = 0.8) -> bool:
+        keywords1 = set(model1.get('keywords', []))
+        keywords2 = set(model2.get('keywords', []))
+        
+        if not keywords1 or not keywords2:
+            return False
+        
+        keyword_similarity = len(keywords1.intersection(keywords2)) / len(keywords1.union(keywords2))
+        
+        responses1 = model1.get('responses', [])
+        responses2 = model2.get('responses', [])
+        
+        response_similarities = []
+        for r1 in responses1[:5]:
+            for r2 in responses2[:5]:
+                sim = similarity_score(r1, r2)
+                response_similarities.append(sim)
+        
+        avg_response_similarity = sum(response_similarities) / len(response_similarities) if response_similarities else 0
+        
+        min_confidence = min(model1.get('confidence', 0), model2.get('confidence', 0))
+        
+        return (keyword_similarity > merge_threshold and 
+                avg_response_similarity > merge_threshold and 
+                min_confidence > 0.7)
+    
+    def merge_mini_models(self, model1: Dict[str, Any], model2: Dict[str, Any]) -> Dict[str, Any]:
+        merged_model = {
+            'patterns': model1.get('patterns', []) + model2.get('patterns', []),
+            'responses': model1.get('responses', []) + model2.get('responses', []),
+            'weights': model1.get('weights', []) + model2.get('weights', []),
+            'confidence': (model1.get('confidence', 0) + model2.get('confidence', 0)) / 2,
+            'grammar_rules': list(set(model1.get('grammar_rules', []) + model2.get('grammar_rules', []))),
+            'keywords': list(set(model1.get('keywords', []) + model2.get('keywords', []))),
+            'training_samples': model1.get('training_samples', 0) + model2.get('training_samples', 0)
+        }
+        
+        return merged_model
+
+
+class AgGPTTrainer:
+    
+    def __init__(self, models_dir: str = "models"):
+        self.feather_manager = FeatherManager(models_dir)
+        self.mini_trainer = MiniModelTrainer(self.feather_manager)
+        self.target_size_mb = 5
+        self.estimated_size_per_pair = 1000
+        self.chunk_size = (self.target_size_mb * 1024 * 1024) // self.estimated_size_per_pair
+    
+    def load_training_data(self, file_path: str) -> List[Tuple[str, str]]:
+        training_pairs = []
+        
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        
+        conversations = content.split('<eos>')
+        
+        print(f"Processing {len(conversations)} conversation chunks...")
+        
+        for conversation in tqdm(conversations, desc="Parsing conversations"):
+            conversation = conversation.strip()
+            if not conversation:
+                continue
+            
+            user_match = re.search(r'user:\s*(.*?)(?=\n<pad>|\nai:|$)', conversation, re.DOTALL)
+            ai_match = re.search(r'ai:\s*(.*?)$', conversation, re.DOTALL)
+            
+            if user_match and ai_match:
+                user_input = user_match.group(1).strip()
+                ai_response = ai_match.group(1).strip()
+                
+                user_input = re.sub(r'<pad>', '', user_input).strip()
+                ai_response = re.sub(r'<pad>', '', ai_response).strip()
+                
+                if user_input and ai_response and len(user_input) > 0 and len(ai_response) > 0:
+                    training_pairs.append((user_input, ai_response))
+        
+        print(f"Extracted {len(training_pairs)} training pairs")
+        return training_pairs
+    
+    def create_training_chunks(self, training_pairs: List[Tuple[str, str]]) -> List[List[Tuple[str, str]]]:
+        shuffled_pairs = training_pairs.copy()
+        random.shuffle(shuffled_pairs)
+        
+        chunks = []
+        total_pairs = len(shuffled_pairs)
+        
+        for i in range(0, total_pairs, self.chunk_size):
+            chunk = shuffled_pairs[i:i + self.chunk_size]
+            if len(chunk) >= 5:
+                chunks.append(chunk)
+        
+        print(f"Created {len(chunks)} training chunks (target: {self.target_size_mb}MB each)")
+        return chunks
+    
+    def train(self, training_file: str = "training_data/corpora.txt", merge_similar: bool = True):
+        print("Starting AgGPT-17 Training with Scalable Feather Architecture")
+        print("=" * 60)
+        
+        cleared_count = self.feather_manager.clear_all_models()
+        if cleared_count > 0:
+            print(f"Cleared {cleared_count} existing models")
+        
+        print("Loading training data...")
+        training_pairs = self.load_training_data(training_file)
+        
+        if not training_pairs:
+            print("No training data found!")
+            return
+        
+        print("Creating training chunks...")
+        training_chunks = self.create_training_chunks(training_pairs)
+        
+        print("Training mini-models...")
+        trained_models = []
+        model_id = 1
+        
+        progress_bar = tqdm(training_chunks, desc="Training mini-models")
+        for chunk in progress_bar:
+            mini_model = self.mini_trainer.train_mini_model(chunk)
+            
+            if mini_model:
+                trained_models.append(mini_model)
+                self.feather_manager.save_mini_model(mini_model, model_id)
+                model_id += 1
+                
+                progress_bar.set_postfix({
+                    'Models': len(trained_models),
+                    'Confidence': f"{mini_model['confidence']:.3f}"
+                })
+        
+        print(f"Trained {len(trained_models)} mini-models")
+        
+        if merge_similar and len(trained_models) > 1:
+            print("Merging similar models...")
+            self._merge_similar_models()
+        
+        final_count = self.feather_manager.get_model_count()
+        print(f"Training complete! Final model count: {final_count}")
+        print("=" * 60)
+    
+    def _merge_similar_models(self):
+        all_models = self.feather_manager.load_all_models()
+        if len(all_models) < 2:
+            return
+        
+        merged_pairs = []
+        models_to_delete = set()
+        
+        print(f"Checking {len(all_models)} models for merging opportunities...")
+        
+        progress_bar = tqdm(range(len(all_models)), desc="Merging models")
+        for i in progress_bar:
+            if i in models_to_delete:
+                continue
+            
+            for j in range(i + 1, len(all_models)):
+                if j in models_to_delete:
+                    continue
+                
+                model1 = all_models[i]
+                model2 = all_models[j]
+                
+                if self.mini_trainer.should_merge_models(model1, model2):
+                    merged_model = self.mini_trainer.merge_mini_models(model1, model2)
+                    
+                    new_id = self.feather_manager.get_next_model_id()
+                    self.feather_manager.save_mini_model(merged_model, new_id)
+                    
+                    models_to_delete.add(i)
+                    models_to_delete.add(j)
+                    merged_pairs.append((model1.get('model_id', i), model2.get('model_id', j), new_id))
+                    
+                    break
+        
+        for model_idx in models_to_delete:
+            if model_idx < len(all_models):
+                model_id = all_models[model_idx].get('model_id', model_idx + 1)
+                self.feather_manager.delete_model(model_id)
+        
+        if merged_pairs:
+            print(f"Merged {len(merged_pairs)} pairs of similar models")
+        else:
+            print("No similar models found for merging")
+
+
+def main():
+    print("AgGPT-17 Scalable Feather Architecture Trainer")
+    print("=" * 50)
+    
+    trainer = AgGPTTrainer()
+    
+    try:
+        trainer.train(merge_similar=True)
+    except KeyboardInterrupt:
+        print("\nTraining interrupted by user")
+    except Exception as e:
+        print(f"Training failed: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

training_data/corpora.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a33906ee116d1b69cc3a67eb9983ddb2fa3d18f2ea8498ca7431f9da829d354
+size 49496569