Update app.py

app.py

@@ -1,625 +1,678 @@
 import gradio as gr
 import requests
 import json
-from datetime import datetime, timedelta
 import re
 import xml.etree.ElementTree as ET
+import numpy as np
 import random
 import hashlib
-import math
-from collections import defaultdict
+from datetime import datetime
+from collections import defaultdict, Counter
+import pickle
+import os
+import threading
+import time
 
-class UniversalAI:
+class TokenPredictor:
     def __init__(self):
-        # Simulated massive knowledge base (trilioni di token)
-        self.knowledge_domains = self.initialize_knowledge_domains()
-        self.conversation_memory = []
-        self.user_profile = {
-            "interests": set(),
-            "expertise_level": "intermediate",
-            "conversation_style": "balanced",
-            "topics_discussed": defaultdict(int)
-        }
-        
-        # Real-time data sources
+        # Token database e vocabulary
+        self.vocabulary = {}  # token_id -> token_string
+        self.token_to_id = {}  # token_string -> token_id
+        self.vocab_size = 0
+        
+        # Neural Network semplificato per predizione
+        self.embedding_dim = 256
+        self.hidden_dim = 512
+        self.context_length = 32
+        
+        # Parametri del network (pesi)
+        self.embeddings = None
+        self.hidden_weights = None
+        self.output_weights = None
+        
+        # Pattern database per apprendimento
+        self.token_patterns = defaultdict(list)  # token -> [next_tokens]
+        self.bigram_counts = defaultdict(Counter)  # token -> {next_token: count}
+        self.trigram_counts = defaultdict(Counter)  # (tok1,tok2) -> {next_token: count}
+        
+        # Dataset sources (pubblici, no API key)
         self.data_sources = {
-            "news": {
-                "reuters": "https://feeds.reuters.com/reuters/worldNews",
-                "bbc": "https://feeds.bbci.co.uk/news/world/rss.xml",
-                "bbc_tech": "https://feeds.bbci.co.uk/news/technology/rss.xml",
-                "bbc_science": "https://feeds.bbci.co.uk/news/science_and_environment/rss.xml"
-            }
+            "gutenberg": "https://www.gutenberg.org/files/",
+            "wikipedia_dumps": "https://dumps.wikimedia.org/enwiki/latest/",
+            "news_rss": [
+                "https://feeds.reuters.com/reuters/worldNews",
+                "https://feeds.bbci.co.uk/news/world/rss.xml",
+                "https://feeds.bbci.co.uk/news/science_and_environment/rss.xml",
+                "https://feeds.bbci.co.uk/news/technology/rss.xml"
+            ],
+            "academic_arxiv": "https://arxiv.org/list/cs/recent",
+            "reddit_json": "https://files.pushshift.io/reddit/",
+            "opensubtitles": "https://opus.nlpl.eu/OpenSubtitles.php",
+            "common_crawl": "https://data.commoncrawl.org/crawl-data/"
         }
         
-        # Advanced reasoning capabilities
-        self.reasoning_frameworks = {
-            "logical": self.logical_reasoning,
-            "creative": self.creative_reasoning,
-            "analytical": self.analytical_reasoning,
-            "synthetic": self.synthetic_reasoning,
-            "critical": self.critical_reasoning
-        }
+        # Data collection stats
+        self.total_tokens_collected = 0
+        self.quality_score_threshold = 0.7
+        self.collection_active = False
         
-        # Multi-domain expertise simulation
-        self.expertise_levels = {
-            "science": 0.95,
-            "technology": 0.92,
-            "history": 0.88,
-            "philosophy": 0.85,
-            "economics": 0.90,
-            "politics": 0.87,
-            "culture": 0.83,
-            "arts": 0.80,
-            "medicine": 0.85,
-            "engineering": 0.88,
-            "psychology": 0.82,
-            "education": 0.84,
-            "environment": 0.86,
-            "business": 0.89
-        }
+        # Training state
+        self.training_loss = []
+        self.epochs_trained = 0
+        self.learning_rate = 0.001
         
-    def initialize_knowledge_domains(self):
-        """Simulates massive pre-training on internet-scale data"""
-        return {
-            "science_and_technology": {
-                "keywords": ["AI", "machine learning", "quantum", "physics", "chemistry", "biology", 
-                           "computer science", "engineering", "mathematics", "astronomy", "genetics",
-                           "nanotechnology", "robotics", "blockchain", "cybersecurity"],
-                "concepts": {
-                    "artificial_intelligence": {
-                        "definition": "Simulation of human intelligence in machines",
-                        "applications": ["autonomous vehicles", "medical diagnosis", "natural language processing"],
-                        "challenges": ["bias", "interpretability", "alignment"],
-                        "future_trends": ["AGI", "quantum AI", "neuromorphic computing"]
-                    },
-                    "quantum_computing": {
-                        "definition": "Computing using quantum mechanical phenomena",
-                        "applications": ["cryptography", "drug discovery", "optimization"],
-                        "challenges": ["decoherence", "error correction", "scalability"],
-                        "future_trends": ["quantum supremacy", "quantum internet", "quantum AI"]
-                    }
-                }
-            },
-            "humanities_and_culture": {
-                "keywords": ["history", "philosophy", "literature", "art", "music", "religion",
-                           "anthropology", "sociology", "linguistics", "archaeology", "ethics"],
-                "concepts": {
-                    "philosophy": {
-                        "branches": ["metaphysics", "epistemology", "ethics", "logic", "aesthetics"],
-                        "major_thinkers": ["Plato", "Aristotle", "Kant", "Nietzsche", "Wittgenstein"],
-                        "contemporary_issues": ["consciousness", "free will", "meaning of life"]
-                    },
-                    "history": {
-                        "periods": ["ancient", "medieval", "renaissance", "modern", "contemporary"],
-                        "themes": ["civilizations", "wars", "revolutions", "cultural movements"],
-                        "methodologies": ["primary sources", "historiography", "comparative analysis"]
-                    }
-                }
-            },
-            "social_sciences": {
-                "keywords": ["psychology", "sociology", "economics", "political science", "anthropology",
-                           "education", "communication", "criminology", "social work"],
-                "concepts": {
-                    "psychology": {
-                        "branches": ["cognitive", "behavioral", "developmental", "clinical", "social"],
-                        "theories": ["cognitive theory", "behaviorism", "psychoanalysis", "humanistic"],
-                        "applications": ["therapy", "education", "organizational behavior"]
-                    },
-                    "economics": {
-                        "schools": ["classical", "keynesian", "chicago", "austrian", "behavioral"],
-                        "concepts": ["supply and demand", "inflation", "GDP", "market efficiency"],
-                        "current_issues": ["inequality", "automation", "cryptocurrency", "sustainability"]
-                    }
-                }
-            },
-            "current_affairs": {
-                "keywords": ["politics", "international relations", "conflicts", "diplomacy", "elections",
-                           "climate change", "pandemics", "migration", "trade", "terrorism"],
-                "concepts": {
-                    "geopolitics": {
-                        "theories": ["realism", "liberalism", "constructivism", "critical theory"],
-                        "actors": ["states", "international organizations", "NGOs", "multinational corporations"],
-                        "issues": ["security", "economic interdependence", "human rights", "sovereignty"]
-                    }
-                }
-            },
-            "practical_skills": {
-                "keywords": ["programming", "project management", "communication", "leadership",
-                           "problem solving", "creativity", "critical thinking", "research"],
-                "concepts": {
-                    "programming": {
-                        "languages": ["Python", "JavaScript", "Java", "C++", "Rust", "Go"],
-                        "paradigms": ["object-oriented", "functional", "procedural", "declarative"],
-                        "applications": ["web development", "data science", "AI/ML", "systems programming"]
-                    }
-                }
-            }
-        }
-    
-    def fetch_real_time_data(self, domain="general"):
-        """Fetches real-time data from multiple sources"""
-        all_data = []
+        self.initialize_network()
         
-        # News sources based on domain
-        sources_to_check = []
-        if domain in ["science", "technology", "general"]:
-            sources_to_check.extend(["reuters", "bbc", "bbc_tech", "bbc_science"])
-        else:
-            sources_to_check.extend(["reuters", "bbc"])
-        
-        for source in sources_to_check:
-            if source in self.data_sources["news"]:
-                try:
-                    response = requests.get(self.data_sources["news"][source], timeout=5)
-                    if response.status_code == 200:
-                        root = ET.fromstring(response.content)
-                        for item in root.findall(".//item")[:3]:
-                            title = item.find("title")
-                            description = item.find("description")
-                            if title is not None:
-                                all_data.append({
-                                    "source": source.upper(),
-                                    "title": title.text,
-                                    "description": description.text if description is not None else "",
-                                    "domain": self.classify_content_domain(title.text),
-                                    "timestamp": datetime.now()
-                                })
-                except:
-                    continue
-        
-        return all_data[:10]
-    
-    def classify_content_domain(self, text):
-        """Classifies content into knowledge domains"""
-        text_lower = text.lower()
-        
-        domain_indicators = {
-            "science_and_technology": ["AI", "technology", "science", "research", "innovation", "quantum", "space"],
-            "current_affairs": ["politics", "election", "government", "conflict", "diplomacy", "war", "crisis"],
-            "social_sciences": ["economy", "market", "society", "culture", "education", "health"],
-            "humanities_and_culture": ["art", "literature", "philosophy", "history", "culture", "religion"]
-        }
+    def initialize_network(self):
+        """Inizializza rete neurale con pesi casuali"""
+        # Embedding layer: converte token_id in vettori densi
+        self.embeddings = np.random.normal(0, 0.1, (50000, self.embedding_dim))
         
-        scores = {}
-        for domain, indicators in domain_indicators.items():
-            score = sum(1 for indicator in indicators if indicator in text_lower)
-            scores[domain] = score
-        
-        return max(scores, key=scores.get) if any(scores.values()) else "general"
-    
-    def detect_query_complexity(self, query):
-        """Analyzes query complexity and required reasoning type"""
-        complexity_indicators = {
-            "simple": ["what is", "define", "quando", "dove", "chi è"],
-            "moderate": ["how does", "why", "explain", "compare", "difference"],
-            "complex": ["analyze", "evaluate", "synthesize", "predict", "implications"],
-            "creative": ["imagine", "create", "design", "invent", "brainstorm"],
-            "philosophical": ["meaning", "purpose", "consciousness", "existence", "truth", "reality"]
-        }
+        # Hidden layer weights  
+        self.hidden_weights = np.random.normal(0, 0.1, (self.embedding_dim * self.context_length, self.hidden_dim))
+        self.hidden_bias = np.zeros(self.hidden_dim)
         
-        query_lower = query.lower()
-        detected_complexity = "moderate"  # default
+        # Output layer weights
+        self.output_weights = np.random.normal(0, 0.1, (self.hidden_dim, 50000))
+        self.output_bias = np.zeros(50000)
         
-        for complexity, indicators in complexity_indicators.items():
-            if any(indicator in query_lower for indicator in indicators):
-                detected_complexity = complexity
+        print("🧠 Neural Network inizializzato con pesi casuali")
+    
+    def collect_quality_data(self, max_tokens=1000000):
+        """Raccoglie dati di qualità da fonti pubbliche"""
+        print("🕷️ Iniziando raccolta dati da fonti pubbliche...")
+        self.collection_active = True
+        collected_texts = []
+        
+        # 1. News RSS feeds (real-time, alta qualità)
+        news_texts = self.scrape_news_feeds()
+        collected_texts.extend(news_texts)
+        print(f"📰 Raccolti {len(news_texts)} articoli news")
+        
+        # 2. Wikipedia abstracts (altissima qualità)
+        wiki_texts = self.scrape_wikipedia_samples()
+        collected_texts.extend(wiki_texts)
+        print(f"📚 Raccolti {len(wiki_texts)} abstract Wikipedia")
+        
+        # 3. ArXiv papers abstracts (qualità accademica)
+        arxiv_texts = self.scrape_arxiv_abstracts()
+        collected_texts.extend(arxiv_texts)
+        print(f"🔬 Raccolti {len(arxiv_texts)} abstract ArXiv")
+        
+        # 4. Project Gutenberg (libri pubblici)
+        gutenberg_texts = self.scrape_gutenberg_samples()
+        collected_texts.extend(gutenberg_texts)
+        print(f"📖 Raccolti {len(gutenberg_texts)} testi Gutenberg")
+        
+        # Quality filtering
+        quality_texts = self.filter_quality_texts(collected_texts)
+        print(f"✅ Filtrati {len(quality_texts)} testi di qualità")
+        
+        # Tokenization
+        all_tokens = []
+        for text in quality_texts:
+            tokens = self.tokenize_text(text)
+            all_tokens.extend(tokens)
+            if len(all_tokens) >= max_tokens:
                 break
         
-        return detected_complexity
+        self.total_tokens_collected = len(all_tokens)
+        print(f"🎯 Raccolti {self.total_tokens_collected:,} token di qualità")
+        
+        # Build vocabulary
+        self.build_vocabulary(all_tokens)
+        
+        # Extract patterns per training
+        self.extract_training_patterns(all_tokens)
+        
+        self.collection_active = False
+        return all_tokens
+    
+    def scrape_news_feeds(self):
+        """Scrape RSS news feeds per contenuto di qualità"""
+        texts = []
+        
+        for rss_url in self.data_sources["news_rss"][:2]:  # Limit per demo
+            try:
+                response = requests.get(rss_url, timeout=5)
+                if response.status_code == 200:
+                    root = ET.fromstring(response.content)
+                    for item in root.findall(".//item")[:5]:
+                        title = item.find("title")
+                        description = item.find("description")
+                        if title is not None:
+                            text = title.text
+                            if description is not None:
+                                text += " " + description.text
+                            texts.append(self.clean_text(text))
+            except:
+                continue
+        
+        return texts
     
-    def extract_topics_and_entities(self, query):
-        """Advanced topic and entity extraction"""
-        # Domain classification
-        domain = self.classify_content_domain(query)
+    def scrape_wikipedia_samples(self):
+        """Scrape Wikipedia content (sample)"""
+        texts = []
         
-        # Entity extraction patterns
-        entities = {
-            "people": re.findall(r'\b[A-Z][a-z]+ [A-Z][a-z]+\b', query),
-            "places": re.findall(r'\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b', query),
-            "concepts": [],
-            "time_references": re.findall(r'\b(?:today|tomorrow|yesterday|next year|future|past|2024|2025)\b', query, re.IGNORECASE)
-        }
+        # Wikipedia API per articoli casuali
+        wiki_api_urls = [
+            "https://en.wikipedia.org/api/rest_v1/page/random/summary",
+            "https://en.wikipedia.org/w/api.php?action=query&format=json&list=random&rnnamespace=0&rnlimit=5"
+        ]
         
-        # Extract concepts based on domain knowledge
-        if domain in self.knowledge_domains:
-            domain_keywords = self.knowledge_domains[domain]["keywords"]
-            for keyword in domain_keywords:
-                if keyword.lower() in query.lower():
-                    entities["concepts"].append(keyword)
-        
-        return {
-            "domain": domain,
-            "entities": entities,
-            "topics": entities["concepts"][:5]  # Top 5 relevant topics
-        }
+        try:
+            for i in range(3):  # 3 articoli casuali
+                response = requests.get(wiki_api_urls[0], timeout=5)
+                if response.status_code == 200:
+                    data = response.json()
+                    if 'extract' in data:
+                        texts.append(self.clean_text(data['extract']))
+        except:
+            pass
+        
+        return texts
     
-    def logical_reasoning(self, query, context):
-        """Simulates logical reasoning processes"""
-        return {
-            "premises": f"Based on established facts about {context.get('domain', 'the topic')}",
-            "inference": "Following logical deduction and evidence",
-            "conclusion": "Therefore, the most logical conclusion is"
-        }
+    def scrape_arxiv_abstracts(self):
+        """Scrape ArXiv abstracts (sample)"""
+        texts = []
+        
+        # ArXiv RSS feed per CS papers
+        arxiv_rss = "http://export.arxiv.org/rss/cs"
+        
+        try:
+            response = requests.get(arxiv_rss, timeout=5)
+            if response.status_code == 200:
+                root = ET.fromstring(response.content)
+                for item in root.findall(".//item")[:3]:
+                    description = item.find("description")
+                    if description is not None:
+                        # Extract abstract from description
+                        desc_text = description.text
+                        if "Abstract:" in desc_text:
+                            abstract = desc_text.split("Abstract:")[1].strip()
+                            texts.append(self.clean_text(abstract))
+        except:
+            pass
+        
+        return texts
     
-    def creative_reasoning(self, query, context):
-        """Simulates creative thinking"""
-        return {
-            "approach": "Thinking outside conventional frameworks",
-            "perspective": "Considering novel connections and possibilities",
-            "innovation": "Generating original insights and solutions"
-        }
+    def scrape_gutenberg_samples(self):
+        """Scrape Project Gutenberg public domain texts (sample)"""
+        texts = []
+        
+        # Sample di testi Gutenberg famosi (public domain)
+        gutenberg_samples = [
+            "https://www.gutenberg.org/files/11/11-0.txt",  # Alice in Wonderland
+            "https://www.gutenberg.org/files/74/74-0.txt",  # Tom Sawyer
+            "https://www.gutenberg.org/files/1342/1342-0.txt",  # Pride and Prejudice
+        ]
+        
+        for url in gutenberg_samples[:1]:  # Solo 1 per demo
+            try:
+                response = requests.get(url, timeout=10)
+                if response.status_code == 200:
+                    text = response.text
+                    # Extract portion of text (primi 5000 chars)
+                    if len(text) > 1000:
+                        sample = text[1000:6000]  # Skip header
+                        texts.append(self.clean_text(sample))
+            except:
+                continue
+        
+        return texts
     
-    def analytical_reasoning(self, query, context):
-        """Simulates analytical breakdown"""
-        return {
-            "decomposition": "Breaking down complex problems into components",
-            "analysis": "Examining each element systematically",
-            "synthesis": "Integrating findings into coherent understanding"
-        }
+    def clean_text(self, text):
+        """Pulisce e normalizza il testo"""
+        if not text:
+            return ""
+        
+        # Remove HTML tags
+        text = re.sub(r'<[^>]+>', ' ', text)
+        
+        # Normalize whitespace
+        text = re.sub(r'\s+', ' ', text)
+        
+        # Remove special characters (keep basic punctuation)
+        text = re.sub(r'[^\w\s\.\,\!\?\;\:\-\(\)\"\']+', ' ', text)
+        
+        # Remove extra spaces
+        text = text.strip()
+        
+        return text
     
-    def synthetic_reasoning(self, query, context):
-        """Simulates synthesis across domains"""
-        return {
-            "integration": "Combining insights from multiple fields",
-            "connections": "Identifying patterns across disciplines",
-            "emergence": "Discovering emergent properties and relationships"
-        }
+    def filter_quality_texts(self, texts):
+        """Filtra testi per qualità"""
+        quality_texts = []
+        
+        for text in texts:
+            score = self.calculate_quality_score(text)
+            if score >= self.quality_score_threshold:
+                quality_texts.append(text)
+        
+        return quality_texts
     
-    def critical_reasoning(self, query, context):
-        """Simulates critical evaluation"""
-        return {
-            "evaluation": "Assessing evidence quality and reliability",
-            "bias_check": "Identifying potential biases and limitations",
-            "alternatives": "Considering alternative explanations and viewpoints"
-        }
+    def calculate_quality_score(self, text):
+        """Calcola score di qualità del testo"""
+        if not text or len(text) < 50:
+            return 0.0
+        
+        score = 0.0
+        
+        # Length score (optimal 100-5000 chars)
+        length = len(text)
+        if 100 <= length <= 5000:
+            score += 0.3
+        elif length > 50:
+            score += 0.1
+        
+        # Language quality (proportion of dictionary words)
+        words = text.lower().split()
+        if words:
+            # Simple English word detection
+            english_words = sum(1 for word in words if self.is_likely_english_word(word))
+            word_ratio = english_words / len(words)
+            score += word_ratio * 0.4
+        
+        # Sentence structure (has proper punctuation)
+        sentences = re.split(r'[.!?]+', text)
+        if len(sentences) > 1:
+            score += 0.2
+        
+        # Avoid repetitive text
+        word_set = set(words) if words else set()
+        if words and len(word_set) / len(words) > 0.5:  # Vocabulary diversity
+            score += 0.1
+        
+        return score
     
-    def generate_comprehensive_response(self, query):
-        """Main response generation with advanced reasoning"""
-        try:
-            # Analyze query
-            complexity = self.detect_query_complexity(query)
-            extraction = self.extract_topics_and_entities(query)
-            domain = extraction["domain"]
-            
-            # Fetch real-time data
-            real_time_data = self.fetch_real_time_data(domain)
-            
-            # Select appropriate reasoning framework
-            reasoning_type = self.select_reasoning_type(complexity, domain)
-            reasoning_process = self.reasoning_frameworks[reasoning_type](query, extraction)
-            
-            # Generate response based on complexity and domain
-            if complexity == "simple":
-                response = self.generate_direct_response(query, extraction, real_time_data)
-            elif complexity == "creative":
-                response = self.generate_creative_response(query, extraction, reasoning_process)
-            elif complexity == "philosophical":
-                response = self.generate_philosophical_response(query, extraction, reasoning_process)
-            else:
-                response = self.generate_analytical_response(query, extraction, real_time_data, reasoning_process)
+    def is_likely_english_word(self, word):
+        """Simple heuristic per English words"""
+        word = re.sub(r'[^\w]', '', word.lower())
+        if len(word) < 2:
+            return False
+        
+        # Basic English patterns
+        common_patterns = [
+            r'^[a-z]+$',  # Only letters
+            r'.*[aeiou].*',  # Contains vowels
+        ]
+        
+        return any(re.match(pattern, word) for pattern in common_patterns)
+    
+    def tokenize_text(self, text):
+        """Tokenizza il testo in token"""
+        # Simple word-based tokenization con punctuation
+        # In produzione: usare BPE (Byte Pair Encoding)
+        
+        # Split on whitespace e punctuation
+        tokens = re.findall(r'\w+|[.!?;,]', text.lower())
+        
+        return tokens
+    
+    def build_vocabulary(self, tokens):
+        """Costruisce vocabulary da tokens"""
+        token_counts = Counter(tokens)
+        
+        # Keep only tokens con frequency >= 2
+        filtered_tokens = {token: count for token, count in token_counts.items() if count >= 2}
+        
+        # Add special tokens
+        vocab_list = ['<PAD>', '<UNK>', '<START>', '<END>'] + list(filtered_tokens.keys())
+        
+        self.vocabulary = {i: token for i, token in enumerate(vocab_list)}
+        self.token_to_id = {token: i for i, token in enumerate(vocab_list)}
+        self.vocab_size = len(vocab_list)
+        
+        print(f"📚 Vocabulary costruito: {self.vocab_size:,} token unici")
+    
+    def extract_training_patterns(self, tokens):
+        """Estrae pattern per training prediction"""
+        print("🔍 Estraendo pattern per training...")
+        
+        # Convert tokens to IDs
+        token_ids = [self.token_to_id.get(token, 1) for token in tokens]  # 1 = <UNK>
+        
+        # Extract bigrams
+        for i in range(len(token_ids) - 1):
+            current_token = token_ids[i]
+            next_token = token_ids[i + 1]
+            self.bigram_counts[current_token][next_token] += 1
+        
+        # Extract trigrams  
+        for i in range(len(token_ids) - 2):
+            context = (token_ids[i], token_ids[i + 1])
+            next_token = token_ids[i + 2]
+            self.trigram_counts[context][next_token] += 1
+        
+        print(f"📊 Pattern estratti:")
+        print(f"   Bigrams: {len(self.bigram_counts):,}")
+        print(f"   Trigrams: {len(self.trigram_counts):,}")
+    
+    def train_neural_network(self, training_sequences, epochs=5):
+        """Training della rete neurale"""
+        print(f"🏋️ Iniziando training per {epochs} epochs...")
+        
+        for epoch in range(epochs):
+            epoch_loss = 0.0
+            batch_count = 0
             
-            # Update user profile
-            self.update_user_profile(query, domain, complexity)
+            # Training su sequenze
+            for i in range(0, len(training_sequences) - self.context_length, 10):
+                # Create input/target pairs
+                input_sequence = training_sequences[i:i + self.context_length]
+                target_token = training_sequences[i + self.context_length]
+                
+                # Forward pass
+                prediction_probs = self.forward_pass(input_sequence)
+                
+                # Calculate loss
+                loss = self.calculate_loss(prediction_probs, target_token)
+                epoch_loss += loss
+                
+                # Backward pass (simplified)
+                self.backward_pass(input_sequence, target_token, prediction_probs)
+                
+                batch_count += 1
+                
+                if batch_count % 100 == 0:
+                    print(f"   Epoch {epoch+1}, Batch {batch_count}, Loss: {loss:.4f}")
             
-            return response
+            avg_loss = epoch_loss / batch_count if batch_count > 0 else 0
+            self.training_loss.append(avg_loss)
+            self.epochs_trained += 1
             
-        except Exception as e:
-            return self.generate_fallback_response(query)
-    
-    def select_reasoning_type(self, complexity, domain):
-        """Selects appropriate reasoning framework"""
-        if complexity == "creative":
-            return "creative"
-        elif complexity == "philosophical":
-            return "critical"
-        elif domain == "science_and_technology":
-            return "analytical"
-        elif complexity == "complex":
-            return "synthetic"
-        else:
-            return "logical"
+            print(f"🎯 Epoch {epoch+1} completato, Loss medio: {avg_loss:.4f}")
+        
+        print("✅ Training completato!")
     
-    def generate_direct_response(self, query, extraction, real_time_data):
-        """Generates direct factual responses"""
-        domain = extraction["domain"]
-        topics = extraction["topics"]
+    def forward_pass(self, input_sequence):
+        """Forward pass della rete neurale"""
+        # Embedding lookup
+        embeddings = np.array([self.embeddings[token_id] for token_id in input_sequence])
+        
+        # Flatten embeddings
+        flattened = embeddings.flatten()
+        
+        # Ensure correct size
+        if len(flattened) < self.embedding_dim * self.context_length:
+            # Pad with zeros
+            padding = np.zeros(self.embedding_dim * self.context_length - len(flattened))
+            flattened = np.concatenate([flattened, padding])
+        else:
+            flattened = flattened[:self.embedding_dim * self.context_length]
         
-        response = []
+        # Hidden layer
+        hidden = np.tanh(np.dot(flattened, self.hidden_weights) + self.hidden_bias)
         
-        # Domain-specific greeting
-        domain_greetings = {
-            "science_and_technology": "Based on current scientific understanding and technological developments,",
-            "current_affairs": "According to the latest information and real-time data,",
-            "social_sciences": "From a social science perspective, drawing on established research,",
-            "humanities_and_culture": "Considering historical and cultural context,"
-        }
+        # Output layer
+        logits = np.dot(hidden, self.output_weights) + self.output_bias
         
-        response.append(domain_greetings.get(domain, "Based on comprehensive analysis,"))
-        response.append("")
+        # Softmax
+        exp_logits = np.exp(logits - np.max(logits))  # Numerical stability
+        probabilities = exp_logits / np.sum(exp_logits)
         
-        # Provide direct answer
-        if topics:
-            response.append(f"**🎯 Regarding {', '.join(topics[:3])}:**")
+        return probabilities
+    
+    def calculate_loss(self, predictions, target_token):
+        """Calcola cross-entropy loss"""
+        # Ensure target_token is in valid range
+        if target_token >= len(predictions):
+            target_token = 1  # <UNK>
+        
+        # Cross-entropy loss
+        return -np.log(predictions[target_token] + 1e-10)  # Small epsilon per numerical stability
+    
+    def backward_pass(self, input_sequence, target_token, predictions):
+        """Simplified backward pass"""
+        # Questo è un backward pass molto semplificato
+        # In produzione: usare autograd frameworks come PyTorch
+        
+        # Calculate gradient per output layer
+        grad_output = predictions.copy()
+        if target_token < len(grad_output):
+            grad_output[target_token] -= 1  # Cross-entropy gradient
+        
+        # Update output weights (simplified)
+        learning_rate = self.learning_rate
+        
+        # Gradient clipping
+        grad_output = np.clip(grad_output, -1.0, 1.0)
+        
+        # Simple weight update (only output layer for demo)
+        if hasattr(self, 'hidden_output'):
+            weight_update = np.outer(self.hidden_output, grad_output)
+            self.output_weights -= learning_rate * weight_update
+    
+    def predict_next_token(self, context_text, num_predictions=5):
+        """Predice i prossimi token dato un contesto"""
+        if not context_text.strip():
+            return ["the", "a", "an", "to", "of"]
+        
+        # Tokenize context
+        context_tokens = self.tokenize_text(context_text)
+        context_ids = [self.token_to_id.get(token, 1) for token in context_tokens]
+        
+        # Use neural network se addestrato
+        if self.epochs_trained > 0 and len(context_ids) > 0:
+            # Take last context_length tokens
+            input_sequence = context_ids[-self.context_length:]
+            if len(input_sequence) < self.context_length:
+                # Pad with <PAD> tokens
+                input_sequence = [0] * (self.context_length - len(input_sequence)) + input_sequence
             
-            # Simulate knowledge retrieval
-            if domain in self.knowledge_domains:
-                domain_concepts = self.knowledge_domains[domain].get("concepts", {})
-                for topic in topics[:2]:
-                    topic_lower = topic.lower()
-                    # Find matching concepts
-                    for concept_key, concept_data in domain_concepts.items():
-                        if topic_lower in concept_key or any(topic_lower in str(v).lower() for v in concept_data.values()):
-                            response.append(f"• **{topic}**: {concept_data.get('definition', 'Key concept in this domain')}")
-                            break
-            response.append("")
-        
-        # Add real-time context if relevant
-        if real_time_data:
-            relevant_news = [item for item in real_time_data if item["domain"] == domain][:2]
-            if relevant_news:
-                response.append("**📡 Current developments:**")
-                for news in relevant_news:
-                    response.append(f"• **[{news['source']}]** {news['title']}")
-                response.append("")
-        
-        response.append("Would you like me to elaborate on any specific aspect?")
-        
-        return "\n".join(response)
-    
-    def generate_creative_response(self, query, extraction, reasoning_process):
-        """Generates creative, innovative responses"""
-        response = []
-        
-        response.append("🎨 **Creative Exploration:**")
-        response.append(f"*{reasoning_process['approach']}*")
-        response.append("")
-        
-        # Creative frameworks
-        if "imagine" in query.lower() or "create" in query.lower():
-            response.extend([
-                "**💡 Innovative Approach:**",
-                "Let me explore this from multiple creative angles:",
-                "",
-                "**🔮 Visionary Perspective:**",
-                "• Imagining radical possibilities beyond current limitations",
-                "• Considering breakthrough innovations and paradigm shifts",
-                "• Exploring unconventional solutions and approaches",
-                "",
-                "**🌟 Creative Synthesis:**",
-                "• Combining disparate ideas in novel ways",
-                "• Drawing inspiration from nature, art, and human experience",
-                "• Challenging assumptions and conventional wisdom",
-                ""
-            ])
-        
-        # Domain-specific creativity
-        domain = extraction["domain"]
-        if domain == "science_and_technology":
-            response.extend([
-                "**🚀 Future-Tech Scenarios:**",
-                "• Breakthrough technologies that could emerge",
-                "• Convergence of multiple scientific fields",
-                "• Transformative applications and societal impacts"
-            ])
-        elif domain == "social_sciences":
-            response.extend([
-                "**🌍 Social Innovation:**",
-                "• Novel social structures and governance models",
-                "• Creative solutions to collective challenges",
-                "• Emerging cultural and behavioral patterns"
-            ])
-        
-        response.append("")
-        response.append("*This creative exploration opens new avenues for thinking about your question.*")
-        
-        return "\n".join(response)
-    
-    def generate_philosophical_response(self, query, extraction, reasoning_process):
-        """Generates deep philosophical responses"""
-        response = []
-        
-        response.append("🤔 **Philosophical Inquiry:**")
-        response.append(f"*{reasoning_process['evaluation']}*")
-        response.append("")
-        
-        # Philosophical frameworks
-        response.extend([
-            "**📚 Multiple Philosophical Perspectives:**",
-            "",
-            "**• Epistemological View:**",
-            "  How do we know what we know about this topic?",
-            "  What are the sources and limits of our understanding?",
-            "",
-            "**• Ethical Considerations:**",
-            "  What moral implications and responsibilities arise?",
-            "  How do we balance competing values and interests?",
-            "",
-            "**• Metaphysical Questions:**",
-            "  What does this reveal about the nature of reality?",
-            "  How does this relate to fundamental questions of existence?",
-            ""
-        ])
-        
-        # Connect to major philosophical traditions
-        response.extend([
-            "**🏛️ Historical Wisdom:**",
-            "• **Ancient Philosophy**: Socratic questioning and Aristotelian analysis",
-            "• **Modern Thought**: Enlightenment rationalism and empiricism",
-            "• **Contemporary Debates**: Current philosophical discourse and emerging paradigms",
-            ""
-        ])
-        
-        response.append("*Philosophy helps us examine not just what we think, but how and why we think it.*")
-        
-        return "\n".join(response)
-    
-    def generate_analytical_response(self, query, extraction, real_time_data, reasoning_process):
-        """Generates comprehensive analytical responses"""
-        domain = extraction["domain"]
-        topics = extraction["topics"]
-        
-        response = []
-        
-        # Analytical framework header
-        response.append("🔬 **Comprehensive Analysis:**")
-        response.append(f"*{reasoning_process['decomposition']}*")
-        response.append("")
-        
-        # Multi-dimensional analysis
-        response.append("**📊 Multi-Dimensional Analysis:**")
-        response.append("")
-        
-        # Domain-specific analysis dimensions
-        if domain == "current_affairs":
-            dimensions = [
-                ("Political Dimension", "Power dynamics, governance structures, and policy implications"),
-                ("Economic Dimension", "Market forces, resource allocation, and financial impacts"),
-                ("Social Dimension", "Cultural factors, public opinion, and societal effects"),
-                ("Historical Context", "Past patterns, precedents, and long-term trends")
-            ]
-        elif domain == "science_and_technology":
-            dimensions = [
-                ("Technical Aspects", "Core mechanisms, capabilities, and limitations"),
-                ("Innovation Potential", "Breakthrough possibilities and future developments"),
-                ("Ethical Implications", "Responsible development and potential risks"),
-                ("Societal Impact", "Transformative effects on daily life and society")
-            ]
-        else:
-            dimensions = [
-                ("Core Components", "Fundamental elements and structures"),
-                ("Interconnections", "Relationships and system dynamics"),
-                ("Implications", "Consequences and broader significance"),
-                ("Future Directions", "Emerging trends and possibilities")
-            ]
-        
-        for dim_name, dim_desc in dimensions:
-            response.append(f"**{dim_name}:**")
-            response.append(f"  {dim_desc}")
-            response.append("")
-        
-        # Evidence from real-time data
-        if real_time_data:
-            response.append("**📡 Current Evidence Base:**")
-            relevant_data = [item for item in real_time_data if item["domain"] == domain][:3]
-            for item in relevant_data:
-                response.append(f"• **[{item['source']}]** {item['title']}")
-            response.append("")
-        
-        # Synthesis and insights
-        response.extend([
-            "**💡 Key Insights:**",
-            f"• **Complexity Level**: High - multiple interacting factors in {domain}",
-            f"• **Certainty Level**: Moderate - based on available evidence and analysis",
-            f"• **Significance**: Important implications for understanding {', '.join(topics[:2]) if topics else 'this topic'}",
-            ""
-        ])
-        
-        # Expert-level considerations
-        if domain in self.expertise_levels:
-            expertise = self.expertise_levels[domain]
-            if expertise > 0.85:
-                response.extend([
-                    "**🎓 Expert-Level Considerations:**",
-                    "• Advanced theoretical frameworks and cutting-edge research",
-                    "• Nuanced understanding of domain-specific methodologies",
-                    "• Integration with interdisciplinary perspectives",
-                    ""
-                ])
-        
-        response.append("*This analysis draws from comprehensive knowledge across multiple disciplines and current data.*")
-        
-        return "\n".join(response)
-    
-    def generate_fallback_response(self, query):
-        """Graceful fallback for complex or unclear queries"""
-        return f"""
-I'm processing your question about "{query[:50]}..." 
-
-While I have extensive knowledge across many domains, I want to provide you with the most accurate and helpful response. 
+            try:
+                prediction_probs = self.forward_pass(input_sequence)
+                
+                # Get top predictions
+                top_indices = np.argsort(prediction_probs)[-num_predictions:][::-1]
+                predictions = []
+                
+                for idx in top_indices:
+                    if idx < len(self.vocabulary):
+                        token = self.vocabulary[idx]
+                        prob = prediction_probs[idx]
+                        predictions.append(f"{token} ({prob:.3f})")
+                
+                return predictions
+            except:
+                pass
+        
+        # Fallback: use pattern matching
+        if len(context_ids) >= 2:
+            # Try trigram
+            last_bigram = (context_ids[-2], context_ids[-1])
+            if last_bigram in self.trigram_counts:
+                most_common = self.trigram_counts[last_bigram].most_common(num_predictions)
+                return [f"{self.vocabulary.get(token_id, '<UNK>')} ({count})" 
+                       for token_id, count in most_common]
+        
+        if len(context_ids) >= 1:
+            # Try bigram
+            last_token = context_ids[-1]
+            if last_token in self.bigram_counts:
+                most_common = self.bigram_counts[last_token].most_common(num_predictions)
+                return [f"{self.vocabulary.get(token_id, '<UNK>')} ({count})" 
+                       for token_id, count in most_common]
+        
+        # Ultimate fallback
+        return ["the", "a", "and", "to", "of"]
+    
+    def get_training_stats(self):
+        """Ritorna statistiche del training"""
+        stats = {
+            "total_tokens": self.total_tokens_collected,
+            "vocabulary_size": self.vocab_size,
+            "epochs_trained": self.epochs_trained,
+            "bigram_patterns": len(self.bigram_counts),
+            "trigram_patterns": len(self.trigram_counts),
+            "current_loss": self.training_loss[-1] if self.training_loss else None,
+            "collection_active": self.collection_active
+        }
+        return stats
 
-Could you help me by:
-• Specifying which aspect interests you most
-• Providing a bit more context about what you're looking for
-• Letting me know if you prefer a technical or general explanation
+# Initialize Token Predictor
+predictor = TokenPredictor()
 
-I can discuss topics ranging from science and technology to philosophy, current affairs, arts, and much more. What would be most valuable for you?
-        """.strip()
-    
-    def update_user_profile(self, query, domain, complexity):
-        """Updates user profile based on interaction"""
-        self.user_profile["topics_discussed"][domain] += 1
-        
-        # Infer interests
-        if self.user_profile["topics_discussed"][domain] > 2:
-            self.user_profile["interests"].add(domain)
-        
-        # Adjust complexity preference
-        if complexity in ["complex", "philosophical"]:
-            if self.user_profile["expertise_level"] == "beginner":
-                self.user_profile["expertise_level"] = "intermediate"
-            elif self.user_profile["expertise_level"] == "intermediate":
-                self.user_profile["expertise_level"] = "advanced"
+def collect_and_train():
+    """Funzione per raccolta dati e training"""
+    try:
+        # Phase 1: Data collection
+        tokens = predictor.collect_quality_data(max_tokens=50000)  # Limit per demo
+        
+        if len(tokens) > 100:
+            # Phase 2: Training
+            predictor.train_neural_network(
+                [predictor.token_to_id.get(token, 1) for token in tokens], 
+                epochs=3
+            )
+            return "✅ Raccolta dati e training completati!"
+        else:
+            return "❌ Dati insufficienti raccolti"
+    except Exception as e:
+        return f"❌ Errore: {str(e)}"
 
-# Initialize Universal AI
-universal_ai = UniversalAI()
+def predict_interface(context_text):
+    """Interface per predizione"""
+    if not context_text.strip():
+        return "Inserisci del testo per ottenere predizioni del prossimo token."
+    
+    predictions = predictor.predict_next_token(context_text)
+    
+    result = f"**🎯 Predizioni per:** '{context_text}'\n\n"
+    result += "**📊 Top token predetti:**\n"
+    for i, pred in enumerate(predictions, 1):
+        result += f"{i}. {pred}\n"
+    
+    # Add stats
+    stats = predictor.get_training_stats()
+    result += f"\n**📈 Stats del modello:**\n"
+    result += f"• Token raccolti: {stats['total_tokens']:,}\n"
+    result += f"• Vocabulary size: {stats['vocabulary_size']:,}\n"
+    result += f"• Epochs addestrati: {stats['epochs_trained']}\n"
+    result += f"• Pattern bigram: {stats['bigram_patterns']:,}\n"
+    result += f"• Pattern trigram: {stats['trigram_patterns']:,}\n"
+    
+    if stats['current_loss']:
+        result += f"• Loss attuale: {stats['current_loss']:.4f}\n"
+    
+    return result
 
-def chat_with_universal_ai(message, history):
-    """Main chat interface"""
-    if not message.strip():
-        return "Hello! I'm a universal AI assistant with knowledge across all domains. I can discuss science, technology, philosophy, current affairs, arts, history, and much more. What would you like to explore today?"
+def get_model_status():
+    """Ritorna status del modello"""
+    stats = predictor.get_training_stats()
+    
+    status = "🤖 **STATUS DEL MODELLO TOKEN PREDICTOR**\n\n"
+    
+    if stats['collection_active']:
+        status += "🔄 **Raccolta dati in corso...**\n\n"
+    elif stats['total_tokens'] == 0:
+        status += "⏳ **Modello non addestrato**\nClicca 'Avvia Training' per iniziare\n\n"
+    else:
+        status += "✅ **Modello addestrato e pronto**\n\n"
+    
+    status += "**📊 Statistiche:**\n"
+    status += f"• **Token raccolti:** {stats['total_tokens']:,}\n"
+    status += f"• **Vocabulary:** {stats['vocabulary_size']:,} token unici\n"
+    status += f"• **Pattern appresi:** {stats['bigram_patterns']:,} bigram, {stats['trigram_patterns']:,} trigram\n"
+    status += f"• **Epochs training:** {stats['epochs_trained']}\n"
+    
+    if stats['current_loss']:
+        status += f"• **Loss attuale:** {stats['current_loss']:.4f}\n"
+    
+    status += "\n**🎯 Capacità:**\n"
+    status += "• Predizione next token da contesto\n"
+    status += "• Pattern recognition da milioni di token\n"
+    status += "• Neural network con embeddings 256D\n"
+    status += "• Training su dati pubblici di qualità\n"
     
-    response = universal_ai.generate_comprehensive_response(message)
-    return response
+    return status
 
-# Advanced Gradio Interface
-iface = gr.ChatInterface(
-    fn=chat_with_universal_ai,
-    title="🧠 Universal AI Assistant",
-    description="""
-    **Advanced AI with Trillion-Token Knowledge & Universal Expertise**
-    
-    I'm designed to be a comprehensive intellectual companion with:
-    
-    🌐 **Universal Knowledge**: Science, technology, philosophy, arts, history, current affairs, and more
-    🧠 **Advanced Reasoning**: Logical, creative, analytical, synthetic, and critical thinking
-    📡 **Real-Time Data**: Current information from global sources
-    🎯 **Adaptive Intelligence**: Adjusts complexity and style to your needs
-    💭 **Deep Analysis**: Multi-dimensional perspective on any topic
-    
-    **I can help with:**
-    • Complex analysis and research
-    • Creative problem-solving
-    • Philosophical discussions
-    • Technical explanations
-    • Current events analysis
-    • Educational content
-    • And virtually any intellectual inquiry
-    
-    *Ask me anything - from quantum physics to poetry, from geopolitics to art history!*
-    """,
-    examples=[
-        "Explain quantum consciousness and its philosophical implications",
-        "Analyze the current state of AI development and future scenarios", 
-        "What are the deeper meanings behind Van Gogh's artistic evolution?",
-        "How do economic theories apply to cryptocurrency markets?",
-        "Explore the relationship between language and thought",
-        "What would happen if we discovered alien intelligence?",
-        "Design a sustainable city for the year 2050",
-        "Explain the philosophy of consciousness to a child"
-    ],
-    theme=gr.themes.Glass(),
-    retry_btn="🔄 Regenerate",
-    undo_btn="↩️ Undo",
-    clear_btn="🧹 New Conversation",
-    submit_btn="🚀 Ask",
-    chatbot=gr.Chatbot(
-        height=500,
-        show_label=False,
-        container=True,
-        bubble_full_width=False,
-        avatar_images=("🧑‍💻", "🧠")
+# Gradio Interface
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    
+    gr.HTML("""
+    <div style="text-align: center; padding: 20px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; border-radius: 10px; margin-bottom: 20px;">
+        <h1>🧠 Token Predictor AI</h1>
+        <p><b>Neural Network che impara a predire il prossimo token</b></p>
+        <p>Input: Milioni di token da database pubblici → Process: Auto-organizzazione neurale → Output: Predizione intelligente</p>
+    </div>
+    """)
+    
+    with gr.Row():
+        with gr.Column(scale=2):
+            gr.HTML("<h3>🎯 Token Prediction</h3>")
+            
+            context_input = gr.Textbox(
+                label="Contesto",
+                placeholder="Es: The capital of France is",
+                lines=2
+            )
+            
+            predict_btn = gr.Button("🔮 Predici Next Token", variant="primary")
+            
+            prediction_output = gr.Textbox(
+                label="Predizioni",
+                lines=10,
+                interactive=False
+            )
+        
+        with gr.Column(scale=1):
+            gr.HTML("<h3>⚙️ Training & Status</h3>")
+            
+            status_output = gr.Textbox(
+                label="Status Modello",
+                lines=15,
+                interactive=False,
+                value=get_model_status()
+            )
+            
+            train_btn = gr.Button("🚀 Avvia Data Collection & Training", variant="secondary")
+            refresh_btn = gr.Button("🔄 Refresh Status", variant="secondary")
+    
+    gr.HTML("""
+    <div style="margin-top: 20px; padding: 15px; background-color: #f0f0f0; border-radius: 8px;">
+        <h4>🔬 Come Funziona:</h4>
+        <ol>
+            <li><b>Data Collection:</b> Raccoglie token da fonti pubbliche (RSS news, Wikipedia, ArXiv, Project Gutenberg)</li>
+            <li><b>Quality Filtering:</b> Filtra contenuti per qualità linguistica e strutturale</li>
+            <li><b>Tokenization:</b> Converte testo in token discreti</li>
+            <li><b>Pattern Extraction:</b> Estrae bigram e trigram per apprendimento</li>
+            <li><b>Neural Training:</b> Addestra rete neurale per predizione next token</li>
+            <li><b>Prediction:</b> Usa pattern appresi per predire token successivi</li>
+        </ol>
+        <p><b>🎯 Obiettivo:</b> AI che predice bene il prossimo token tramite auto-organizzazione neurale su milioni di esempi!</p>
+    </div>
+    """)
+    
+    # Examples
+    gr.Examples(
+        examples=[
+            "The weather today is",
+            "Artificial intelligence will",
+            "The capital of Italy is",
+            "Machine learning algorithms",
+            "In the year 2030",
+            "The most important thing"
+        ],
+        inputs=context_input
+    )
+    
+    # Event handlers
+    predict_btn.click(
+        predict_interface,
+        inputs=[context_input],
+        outputs=[prediction_output]
+    )
+    
+    train_btn.click(
+        collect_and_train,
+        outputs=[status_output]
+    )
+    
+    refresh_btn.click(
+        get_model_status,
+        outputs=[status_output]
     )
-)
 
 if __name__ == "__main__":
-    iface.launch()
+    demo.launch()