|
|
import os |
|
|
import re |
|
|
import random |
|
|
import math |
|
|
from typing import List, Dict, Tuple, Set, Any |
|
|
from collections import defaultdict, Counter |
|
|
import pandas as pd |
|
|
from tqdm import tqdm |
|
|
|
|
|
from feather import FeatherManager, similarity_score, calculate_confidence_score |
|
|
|
|
|
class GrammarRules: |
|
|
|
|
|
@staticmethod |
|
|
def apply_all_rules(text: str) -> str: |
|
|
if not text: |
|
|
return text |
|
|
|
|
|
return text.strip() |
|
|
|
|
|
class PatternExtractor: |
|
|
|
|
|
def __init__(self): |
|
|
self.stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'can', 'shall'} |
|
|
|
|
|
def extract_keywords(self, text: str) -> List[str]: |
|
|
if not text: |
|
|
return [] |
|
|
|
|
|
words = re.findall(r'\b[a-zA-Z]+\b', text.lower()) |
|
|
|
|
|
keywords = [word for word in words if word not in self.stop_words and len(word) > 2] |
|
|
|
|
|
return list(set(keywords)) |
|
|
|
|
|
def create_pattern(self, user_input: str) -> str: |
|
|
if not user_input: |
|
|
return "" |
|
|
|
|
|
pattern = re.sub(r'\s+', ' ', user_input.strip().lower()) |
|
|
|
|
|
pattern = f" {pattern} " |
|
|
|
|
|
return pattern |
|
|
|
|
|
def calculate_pattern_similarity(self, pattern1: str, pattern2: str) -> float: |
|
|
return similarity_score(pattern1.strip(), pattern2.strip()) |
|
|
|
|
|
|
|
|
class MiniModelTrainer: |
|
|
|
|
|
def __init__(self, feather_manager: FeatherManager): |
|
|
self.feather_manager = feather_manager |
|
|
self.pattern_extractor = PatternExtractor() |
|
|
self.grammar_rules = GrammarRules() |
|
|
|
|
|
def train_mini_model(self, training_pairs: List[Tuple[str, str]], confidence_threshold: float = 0.1) -> Dict[str, Any]: |
|
|
if not training_pairs or len(training_pairs) < 2: |
|
|
return None |
|
|
|
|
|
patterns = [] |
|
|
responses = [] |
|
|
weights = [] |
|
|
all_keywords = [] |
|
|
|
|
|
for user_input, ai_response in training_pairs: |
|
|
processed_response = ai_response.strip() |
|
|
|
|
|
pattern = self.pattern_extractor.create_pattern(user_input) |
|
|
|
|
|
keywords = self.pattern_extractor.extract_keywords(user_input) |
|
|
all_keywords.extend(keywords) |
|
|
|
|
|
patterns.append(pattern) |
|
|
responses.append(processed_response) |
|
|
weights.append(1.0) |
|
|
|
|
|
confidence = min(0.9, len(training_pairs) / 20.0) |
|
|
|
|
|
keyword_counter = Counter(all_keywords) |
|
|
top_keywords = [word for word, count in keyword_counter.most_common(10)] |
|
|
|
|
|
mini_model = { |
|
|
'patterns': patterns, |
|
|
'responses': responses, |
|
|
'weights': weights, |
|
|
'confidence': confidence, |
|
|
'grammar_rules': [], |
|
|
'keywords': top_keywords, |
|
|
'training_samples': len(training_pairs) |
|
|
} |
|
|
|
|
|
return mini_model |
|
|
|
|
|
def should_merge_models(self, model1: Dict[str, Any], model2: Dict[str, Any], merge_threshold: float = 0.8) -> bool: |
|
|
keywords1 = set(model1.get('keywords', [])) |
|
|
keywords2 = set(model2.get('keywords', [])) |
|
|
|
|
|
if not keywords1 or not keywords2: |
|
|
return False |
|
|
|
|
|
keyword_similarity = len(keywords1.intersection(keywords2)) / len(keywords1.union(keywords2)) |
|
|
|
|
|
responses1 = model1.get('responses', []) |
|
|
responses2 = model2.get('responses', []) |
|
|
|
|
|
response_similarities = [] |
|
|
for r1 in responses1[:5]: |
|
|
for r2 in responses2[:5]: |
|
|
sim = similarity_score(r1, r2) |
|
|
response_similarities.append(sim) |
|
|
|
|
|
avg_response_similarity = sum(response_similarities) / len(response_similarities) if response_similarities else 0 |
|
|
|
|
|
min_confidence = min(model1.get('confidence', 0), model2.get('confidence', 0)) |
|
|
|
|
|
return (keyword_similarity > merge_threshold and |
|
|
avg_response_similarity > merge_threshold and |
|
|
min_confidence > 0.7) |
|
|
|
|
|
def merge_mini_models(self, model1: Dict[str, Any], model2: Dict[str, Any]) -> Dict[str, Any]: |
|
|
merged_model = { |
|
|
'patterns': model1.get('patterns', []) + model2.get('patterns', []), |
|
|
'responses': model1.get('responses', []) + model2.get('responses', []), |
|
|
'weights': model1.get('weights', []) + model2.get('weights', []), |
|
|
'confidence': (model1.get('confidence', 0) + model2.get('confidence', 0)) / 2, |
|
|
'grammar_rules': list(set(model1.get('grammar_rules', []) + model2.get('grammar_rules', []))), |
|
|
'keywords': list(set(model1.get('keywords', []) + model2.get('keywords', []))), |
|
|
'training_samples': model1.get('training_samples', 0) + model2.get('training_samples', 0) |
|
|
} |
|
|
|
|
|
return merged_model |
|
|
|
|
|
|
|
|
class AgGPTTrainer: |
|
|
|
|
|
def __init__(self, models_dir: str = "models"): |
|
|
self.feather_manager = FeatherManager(models_dir) |
|
|
self.mini_trainer = MiniModelTrainer(self.feather_manager) |
|
|
self.target_size_mb = 5 |
|
|
self.estimated_size_per_pair = 1000 |
|
|
self.chunk_size = (self.target_size_mb * 1024 * 1024) // self.estimated_size_per_pair |
|
|
|
|
|
def load_training_data(self, file_path: str) -> List[Tuple[str, str]]: |
|
|
training_pairs = [] |
|
|
|
|
|
with open(file_path, 'r', encoding='utf-8') as f: |
|
|
content = f.read() |
|
|
|
|
|
conversations = content.split('<eos>') |
|
|
|
|
|
print(f"Processing {len(conversations)} conversation chunks...") |
|
|
|
|
|
for conversation in tqdm(conversations, desc="Parsing conversations"): |
|
|
conversation = conversation.strip() |
|
|
if not conversation: |
|
|
continue |
|
|
|
|
|
user_match = re.search(r'user:\s*(.*?)(?=\n<pad>|\nai:|$)', conversation, re.DOTALL) |
|
|
ai_match = re.search(r'ai:\s*(.*?)$', conversation, re.DOTALL) |
|
|
|
|
|
if user_match and ai_match: |
|
|
user_input = user_match.group(1).strip() |
|
|
ai_response = ai_match.group(1).strip() |
|
|
|
|
|
user_input = re.sub(r'<pad>', '', user_input).strip() |
|
|
ai_response = re.sub(r'<pad>', '', ai_response).strip() |
|
|
|
|
|
if user_input and ai_response and len(user_input) > 0 and len(ai_response) > 0: |
|
|
training_pairs.append((user_input, ai_response)) |
|
|
|
|
|
print(f"Extracted {len(training_pairs)} training pairs") |
|
|
return training_pairs |
|
|
|
|
|
def create_training_chunks(self, training_pairs: List[Tuple[str, str]]) -> List[List[Tuple[str, str]]]: |
|
|
shuffled_pairs = training_pairs.copy() |
|
|
random.shuffle(shuffled_pairs) |
|
|
|
|
|
chunks = [] |
|
|
total_pairs = len(shuffled_pairs) |
|
|
|
|
|
for i in range(0, total_pairs, self.chunk_size): |
|
|
chunk = shuffled_pairs[i:i + self.chunk_size] |
|
|
if len(chunk) >= 5: |
|
|
chunks.append(chunk) |
|
|
|
|
|
print(f"Created {len(chunks)} training chunks (target: {self.target_size_mb}MB each)") |
|
|
return chunks |
|
|
|
|
|
def train(self, training_file: str = "training_data/corpora.txt", merge_similar: bool = True): |
|
|
print("Starting AgGPT-17 Training with Scalable Feather Architecture") |
|
|
print("=" * 60) |
|
|
|
|
|
cleared_count = self.feather_manager.clear_all_models() |
|
|
if cleared_count > 0: |
|
|
print(f"Cleared {cleared_count} existing models") |
|
|
|
|
|
print("Loading training data...") |
|
|
training_pairs = self.load_training_data(training_file) |
|
|
|
|
|
if not training_pairs: |
|
|
print("No training data found!") |
|
|
return |
|
|
|
|
|
print("Creating training chunks...") |
|
|
training_chunks = self.create_training_chunks(training_pairs) |
|
|
|
|
|
print("Training mini-models...") |
|
|
trained_models = [] |
|
|
model_id = 1 |
|
|
|
|
|
progress_bar = tqdm(training_chunks, desc="Training mini-models") |
|
|
for chunk in progress_bar: |
|
|
mini_model = self.mini_trainer.train_mini_model(chunk) |
|
|
|
|
|
if mini_model: |
|
|
trained_models.append(mini_model) |
|
|
self.feather_manager.save_mini_model(mini_model, model_id) |
|
|
model_id += 1 |
|
|
|
|
|
progress_bar.set_postfix({ |
|
|
'Models': len(trained_models), |
|
|
'Confidence': f"{mini_model['confidence']:.3f}" |
|
|
}) |
|
|
|
|
|
print(f"Trained {len(trained_models)} mini-models") |
|
|
|
|
|
if merge_similar and len(trained_models) > 1: |
|
|
print("Merging similar models...") |
|
|
self._merge_similar_models() |
|
|
|
|
|
final_count = self.feather_manager.get_model_count() |
|
|
print(f"Training complete! Final model count: {final_count}") |
|
|
print("=" * 60) |
|
|
|
|
|
def _merge_similar_models(self): |
|
|
all_models = self.feather_manager.load_all_models() |
|
|
if len(all_models) < 2: |
|
|
return |
|
|
|
|
|
merged_pairs = [] |
|
|
models_to_delete = set() |
|
|
|
|
|
print(f"Checking {len(all_models)} models for merging opportunities...") |
|
|
|
|
|
progress_bar = tqdm(range(len(all_models)), desc="Merging models") |
|
|
for i in progress_bar: |
|
|
if i in models_to_delete: |
|
|
continue |
|
|
|
|
|
for j in range(i + 1, len(all_models)): |
|
|
if j in models_to_delete: |
|
|
continue |
|
|
|
|
|
model1 = all_models[i] |
|
|
model2 = all_models[j] |
|
|
|
|
|
if self.mini_trainer.should_merge_models(model1, model2): |
|
|
merged_model = self.mini_trainer.merge_mini_models(model1, model2) |
|
|
|
|
|
new_id = self.feather_manager.get_next_model_id() |
|
|
self.feather_manager.save_mini_model(merged_model, new_id) |
|
|
|
|
|
models_to_delete.add(i) |
|
|
models_to_delete.add(j) |
|
|
merged_pairs.append((model1.get('model_id', i), model2.get('model_id', j), new_id)) |
|
|
|
|
|
break |
|
|
|
|
|
for model_idx in models_to_delete: |
|
|
if model_idx < len(all_models): |
|
|
model_id = all_models[model_idx].get('model_id', model_idx + 1) |
|
|
self.feather_manager.delete_model(model_id) |
|
|
|
|
|
if merged_pairs: |
|
|
print(f"Merged {len(merged_pairs)} pairs of similar models") |
|
|
else: |
|
|
print("No similar models found for merging") |
|
|
|
|
|
|
|
|
def main(): |
|
|
print("AgGPT-17 Scalable Feather Architecture Trainer") |
|
|
print("=" * 50) |
|
|
|
|
|
trainer = AgGPTTrainer() |
|
|
|
|
|
try: |
|
|
trainer.train(merge_similar=True) |
|
|
except KeyboardInterrupt: |
|
|
print("\nTraining interrupted by user") |
|
|
except Exception as e: |
|
|
print(f"Training failed: {e}") |
|
|
import traceback |
|
|
traceback.print_exc() |
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
main() |
|
|
|
