Update app.py

app.py

@@ -1,7 +1,9 @@
+
 import os
 import json
 import time
 import threading
+import random
 from datetime import datetime
 from flask import Flask, request, jsonify, send_from_directory
 from flask_cors import CORS
@@ -24,10 +26,36 @@ LEVEL_1_MAX = 20  # Max entries before archiving to Level 2
 LEVEL_2_MAX = 50  # Max entries before archiving to Level 3
 AUTO_ARCHIVE_ENABLED = True
 AUTO_ARCHIVE_INTERVAL = 600  # 10 minutes
+AUTO_TRAINING_INTERVAL = 300  # 5 minutes - auto-generate training examples
+AUTO_SAVE_ENABLED = True  # Always save responses immediately
 
 # Memory buffer
 MEMORY_BUFFER = []
 
+# Auto-training prompts for diverse learning
+AUTO_TRAINING_PROMPTS = [
+    "Explain quantum computing in simple terms",
+    "How does machine learning work?",
+    "What are the best practices for web development?",
+    "Write a Python function to sort a list",
+    "Explain the difference between AI and ML",
+    "How do neural networks learn?",
+    "What is the future of artificial intelligence?",
+    "Create a creative story about space exploration",
+    "Explain blockchain technology",
+    "How does natural language processing work?",
+    "What are design patterns in software engineering?",
+    "Explain the concept of recursion with examples",
+    "How do databases manage concurrent transactions?",
+    "What is the difference between REST and GraphQL?",
+    "Explain cloud computing and its benefits",
+    "How does encryption work?",
+    "What are the principles of good UI/UX design?",
+    "Explain the concept of Big O notation",
+    "How do search engines rank websites?",
+    "What is containerization and why is it useful?"
+]
+
 def load_training_dataset():
     """Load existing training data from HuggingFace"""
     try:
@@ -82,7 +110,6 @@ def call_openai_gpt4o_mini(prompt):
 def summarize_conversation(query, response):
     """Create a summary of the conversation using AI"""
     if not client:
-        # Fallback summary without AI
         return f"{query[:50]}... -> {response[:50]}..."
 
     try:
@@ -98,7 +125,7 @@ def create_super_summary(summaries):
         return "General knowledge compilation"
 
     try:
-        combined = "\n".join(summaries[:10])  # Use last 10 summaries
+        combined = "\n".join(summaries[:10])
         prompt = f"Create a brief summary of these conversation topics:\n{combined}"
         super_summary = call_openai_gpt4o_mini(prompt)
         return super_summary if super_summary else "General knowledge compilation"
@@ -107,8 +134,6 @@ def create_super_summary(summaries):
 
 def archive_level_1_to_level_2():
     """Archive Level 1 entries to Level 2 with compression"""
-    global MEMORY_BUFFER
-
     dataset = load_training_dataset()
     level_1_entries = [d for d in dataset if d.get("level") == 1 and not d.get("archived")]
 
@@ -131,7 +156,6 @@ def archive_level_1_to_level_2():
         }
         archived_entries.append(archived_entry)
 
-    # Update dataset
     updated_dataset = [d for d in dataset if d.get("level") != 1 or d.get("archived")]
     updated_dataset.extend(archived_entries)
 
@@ -164,7 +188,6 @@ def archive_level_2_to_level_3():
         "entries_compressed": len(level_2_entries)
     }
 
-    # Keep only Level 1, Level 3, and recent Level 2 entries
     updated_dataset = [d for d in dataset if d.get("level") != 2]
     updated_dataset.append(level_3_entry)
 
@@ -173,6 +196,55 @@ def archive_level_2_to_level_3():
         dataset_obj.push_to_hub(TRAINING_DATASET, token=HF_TOKEN, private=False)
         print(f"✅ Created Level 3 super-summary")
 
+def auto_training_worker():
+    """Background worker that automatically generates training examples"""
+    print("🤖 Auto-training worker started")
+    training_count = 0
+
+    while True:
+        try:
+            if not HF_TOKEN or not client:
+                print("⚠️ Auto-training paused - need HF_TOKEN and OPENAI_API_KEY")
+                time.sleep(AUTO_TRAINING_INTERVAL)
+                continue
+
+            # Select random training prompt
+            prompt = random.choice(AUTO_TRAINING_PROMPTS)
+            print(f"\n🎓 Auto-training #{training_count + 1}: '{prompt[:50]}...'")
+
+            # Generate response
+            response = call_openai_gpt4o_mini(prompt)
+            if not response:
+                print("❌ Failed to generate response")
+                time.sleep(AUTO_TRAINING_INTERVAL)
+                continue
+
+            # Create summary
+            summary = summarize_conversation(prompt, response)
+
+            # Create Level 1 memory entry
+            memory_entry = {
+                "query": prompt,
+                "response": response,
+                "summary": summary,
+                "archived": False,
+                "level": 1,
+                "counter": 1,
+                "timestamp": datetime.now().isoformat(),
+                "auto_generated": True
+            }
+
+            # Save immediately to dataset
+            if save_to_training_dataset([memory_entry]):
+                training_count += 1
+                print(f"✅ Auto-trained and saved! Total auto-training: {training_count}")
+
+            time.sleep(AUTO_TRAINING_INTERVAL)
+
+        except Exception as e:
+            print(f"❌ Auto-training error: {e}")
+            time.sleep(AUTO_TRAINING_INTERVAL)
+
 def auto_archive_worker():
     """Background worker that automatically archives memory levels"""
     print("🗄️ Auto-archive worker started")
@@ -197,9 +269,7 @@ def index():
 
 @app.route("/api/chat", methods=["POST"])
 def chat():
-    """Handle chat requests with hierarchical memory learning"""
-    global MEMORY_BUFFER
-
+    """Handle chat requests with automatic dataset saving"""
     data = request.get_json()
     query = data.get("query", "").strip()
 
@@ -227,22 +297,25 @@ def chat():
         "archived": False,
         "level": 1,
         "counter": 1,
-        "timestamp": datetime.now().isoformat()
+        "timestamp": datetime.now().isoformat(),
+        "auto_generated": False
     }
 
-    MEMORY_BUFFER.append(memory_entry)
-
-    # Save to HuggingFace if we have enough entries
-    if len(MEMORY_BUFFER) >= 5:
-        if save_to_training_dataset(MEMORY_BUFFER.copy()):
-            MEMORY_BUFFER.clear()
+    # AUTO-SAVE: Save immediately to HuggingFace
+    if AUTO_SAVE_ENABLED and HF_TOKEN:
+        save_to_training_dataset([memory_entry])
+        saved_status = "saved"
+    else:
+        MEMORY_BUFFER.append(memory_entry)
+        saved_status = "buffered"
 
     return jsonify({
         "success": True,
         "response": response,
         "summary": summary,
         "level": 1,
-        "buffered": len(MEMORY_BUFFER)
+        "buffered": len(MEMORY_BUFFER),
+        "saved": saved_status
     })
 
 @app.route("/api/dataset-stats", methods=["GET"])
@@ -254,6 +327,7 @@ def dataset_stats():
         level_1 = [d for d in training_data if d.get("level") == 1]
         level_2 = [d for d in training_data if d.get("level") == 2]
         level_3 = [d for d in training_data if d.get("level") == 3]
+        auto_generated = [d for d in training_data if d.get("auto_generated")]
 
         return jsonify({
             "success": True,
@@ -262,6 +336,7 @@ def dataset_stats():
             "level_2_archived": len(level_2),
             "level_3_super": len(level_3),
             "buffered": len(MEMORY_BUFFER),
+            "auto_generated_count": len(auto_generated),
             "dataset_url": f"https://huggingface.co/datasets/{TRAINING_DATASET}"
         })
     except Exception as e:
@@ -298,7 +373,7 @@ def archive_now():
         return jsonify({"error": str(e)}), 500
 
 if __name__ == "__main__":
-    port = int(os.getenv("PORT", 7860))
+    port = int(os.getenv("PORT", 5000))
 
     print("🚀 Starting ContinuumGPT Hierarchical Memory System...")
     print(f"📊 Training Dataset: {TRAINING_DATASET}")
@@ -309,14 +384,22 @@ if __name__ == "__main__":
     print(f"   Level 2 (Archived): Compressed summaries (max {LEVEL_2_MAX} before archiving)")
     print(f"   Level 3 (Super): Global knowledge compilation")
     print(f"   Auto-archiving: Every {AUTO_ARCHIVE_INTERVAL} seconds")
+    print(f"   Auto-training: Every {AUTO_TRAINING_INTERVAL} seconds")
+    print(f"   Auto-save: {'ENABLED' if AUTO_SAVE_ENABLED else 'DISABLED'}")
     print("")
 
     if OPENAI_API_KEY:
         print("✅ OpenAI API Key Configured")
+        
         # Start auto-archive worker
         archive_thread = threading.Thread(target=auto_archive_worker, daemon=True)
         archive_thread.start()
         print("✅ Auto-archive worker started")
+        
+        # Start auto-training worker
+        training_thread = threading.Thread(target=auto_training_worker, daemon=True)
+        training_thread.start()
+        print("✅ Auto-training worker started - will generate examples every 5 minutes")
     else:
         print("⚠️ OpenAI API Key Missing - Add OPENAI_API_KEY to enable")
 
@@ -328,8 +411,11 @@ if __name__ == "__main__":
             level = d.get("level", 1)
             level_counts[level] = level_counts.get(level, 0) + 1
         print(f"📚 Current dataset: L1={level_counts[1]}, L2={level_counts[2]}, L3={level_counts[3]}")
+        
+        if AUTO_SAVE_ENABLED:
+            print("💾 Auto-save ENABLED - all responses saved immediately to dataset")
     else:
         print("⚠️ HuggingFace Integration Disabled - Add HF_TOKEN to enable")
 
     print("")
-    app.run(host="0.0.0.0", port=port, debug=False, threaded=True)
+    app.run(host="0.0.0.0", port=port, debug=False, threaded=True)