Update app.py

app.py

@@ -8,7 +8,7 @@ from datetime import datetime
 from typing import List, Dict, Optional, Union
 from concurrent.futures import ThreadPoolExecutor
 
-from fastapi import FastAPI, HTTPException, Request, UploadFile, File, WebSocket, WebSocketDisconnect
+from fastapi import FastAPI, HTTPException, Request, UploadFile, File, WebSocket, WebSocketDisconnect, Depends
 from fastapi.responses import StreamingResponse, HTMLResponse, FileResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
@@ -29,13 +29,25 @@ from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
 from tensorflow.keras.models import load_model
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
+from transformers import BitsAndBytesConfig
 
 # التهيئة الأساسية
 DetectorFactory.seed = 0
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("MarkAI")
 
-app = FastAPI(title="MarkAI - الذكاء الاصطناعي المتكامل", version="2.0")
+app = FastAPI(
+    title="MarkAI - الذكاء الاصطناعي المتكامل",
+    version="2.0",
+    description="منصة متكاملة للذكاء الاصطناعي تدعم توليد النصوص، الأكواد، الصور والفيديوهات مع نظام ذاكرة متقدم",
+    contact={
+        "name": "Ibrahim Lasfar",
+        "email": "ibrahim@markai.com"
+    },
+    license_info={
+        "name": "MIT License",
+    }
+)
 
 # إعدادات CORS
 app.add_middleware(
@@ -53,8 +65,9 @@ os.makedirs("memory/projects", exist_ok=True)
 os.makedirs("memory/code", exist_ok=True)
 os.makedirs("memory/backups", exist_ok=True)
 
-# 1. نماذج اللغات المدعومة (محدثة)
+# 1. نماذج اللغات المدعومة (محدثة مع النماذج الكبيرة)
 LANGUAGE_MODELS = {
+    # النماذج الصغيرة (افتراضية)
     "en": "gpt2-medium",
     "ar": "arbml/gpt2-arabic-poetry",
     "zh": "bert-base-chinese",
@@ -63,7 +76,12 @@ LANGUAGE_MODELS = {
     "de": "dbmdz/gpt2-german",
     "it": "LorenzoDeMattei/GePpeTto",
     "hi": "surajpai/GPT2-Hindi",
-    "code": "codeparrot/codeparrot-small"
+    "code": "codeparrot/codeparrot-small",
+    
+    # النماذج الكبيرة
+    "en-large": "EleutherAI/gpt-j-6B",
+    "ar-large": "bigscience/bloom-7b1",
+    "code-large": "tiiuae/falcon-7b"
 }
 
 # 2. نظام الأمان والمفاتيح
@@ -74,7 +92,7 @@ def load_api_keys():
         with open("memory/api_keys.json", "r") as f:
             return json.load(f)
     except:
-        return {}
+        return {"demo_key": "demo123"}  # مفتاح تجريبي افتراضي
 
 def save_api_keys(keys):
     with open("memory/api_keys.json", "w") as f:
@@ -94,22 +112,25 @@ class AIMemory:
         
     def load_all_data(self):
         """تحميل جميع البيانات من الملفات"""
-        # تحميل المحادثات
-        for conv_file in os.listdir("memory/conversations"):
-            if conv_file.endswith(".json"):
-                conv_id = conv_file.split(".")[0]
-                with open(f"memory/conversations/{conv_file}", "r", encoding="utf-8") as f:
-                    self.conversations[conv_id] = json.load(f)
-        
-        # تحميل المشاريع
-        if os.path.exists("memory/projects/projects.json"):
-            with open("memory/projects/projects.json", "r", encoding="utf-8") as f:
-                self.projects = json.load(f)
-        
-        # تحميل مستودع الأكواد
-        if os.path.exists("memory/code/code_repository.json"):
-            with open("memory/code/code_repository.json", "r", encoding="utf-8") as f:
-                self.code_repository = json.load(f)
+        try:
+            # تحميل المحادثات
+            for conv_file in os.listdir("memory/conversations"):
+                if conv_file.endswith(".json"):
+                    conv_id = conv_file.split(".")[0]
+                    with open(f"memory/conversations/{conv_file}", "r", encoding="utf-8") as f:
+                        self.conversations[conv_id] = json.load(f)
+            
+            # تحميل المشاريع
+            if os.path.exists("memory/projects/projects.json"):
+                with open("memory/projects/projects.json", "r", encoding="utf-8") as f:
+                    self.projects = json.load(f)
+            
+            # تحميل مستودع الأكواد
+            if os.path.exists("memory/code/code_repository.json"):
+                with open("memory/code/code_repository.json", "r", encoding="utf-8") as f:
+                    self.code_repository = json.load(f)
+        except Exception as e:
+            logger.error(f"Error loading data: {str(e)}")
     
     def create_conversation(self, initial_prompt: str) -> str:
         """إنشاء محادثة جديدة مع تسمية تلقائية"""
@@ -133,8 +154,11 @@ class AIMemory:
     def save_conversation(self, conv_id: str):
         """حفظ محادثة معينة"""
         if conv_id in self.conversations:
-            with open(f"memory/conversations/{conv_id}.json", "w", encoding="utf-8") as f:
-                json.dump(self.conversations[conv_id], f, ensure_ascii=False, indent=2)
+            try:
+                with open(f"memory/conversations/{conv_id}.json", "w", encoding="utf-8") as f:
+                    json.dump(self.conversations[conv_id], f, ensure_ascii=False, indent=2)
+            except Exception as e:
+                logger.error(f"Error saving conversation {conv_id}: {str(e)}")
     
     def add_message(self, conv_id: str, role: str, content: str, metadata: dict = {}):
         """إضافة رسالة إلى المحادثة"""
@@ -181,8 +205,11 @@ class AIMemory:
     
     def save_projects(self):
         """حفظ جميع المشاريع"""
-        with open("memory/projects/projects.json", "w", encoding="utf-8") as f:
-            json.dump(self.projects, f, ensure_ascii=False, indent=2)
+        try:
+            with open("memory/projects/projects.json", "w", encoding="utf-8") as f:
+                json.dump(self.projects, f, ensure_ascii=False, indent=2)
+        except Exception as e:
+            logger.error(f"Error saving projects: {str(e)}")
     
     def save_code_snippet(self, code: str, language: str, purpose: str, metadata: dict = {}):
         """حفظ جزء من الكود في المستودع"""
@@ -204,8 +231,11 @@ class AIMemory:
     
     def save_code_repository(self):
         """حفظ مستودع الأكواد"""
-        with open("memory/code/code_repository.json", "w", encoding="utf-8") as f:
-            json.dump(self.code_repository, f, ensure_ascii=False, indent=2)
+        try:
+            with open("memory/code/code_repository.json", "w", encoding="utf-8") as f:
+                json.dump(self.code_repository, f, ensure_ascii=False, indent=2)
+        except Exception as e:
+            logger.error(f"Error saving code repository: {str(e)}")
     
     def backup_data(self):
         """إنشاء نسخة احتياطية لجميع البيانات"""
@@ -213,21 +243,25 @@ class AIMemory:
         backup_dir = f"memory/backups/{timestamp}"
         os.makedirs(backup_dir, exist_ok=True)
         
-        # نسخ المحادثات
-        os.makedirs(f"{backup_dir}/conversations", exist_ok=True)
-        for conv_id, conv_data in self.conversations.items():
-            with open(f"{backup_dir}/conversations/{conv_id}.json", "w", encoding="utf-8") as f:
-                json.dump(conv_data, f, ensure_ascii=False, indent=2)
-        
-        # نسخ المشاريع
-        with open(f"{backup_dir}/projects.json", "w", encoding="utf-8") as f:
-            json.dump(self.projects, f, ensure_ascii=False, indent=2)
-        
-        # نسخ الأكواد
-        with open(f"{backup_dir}/code_repository.json", "w", encoding="utf-8") as f:
-            json.dump(self.code_repository, f, ensure_ascii=False, indent=2)
-        
-        return backup_dir
+        try:
+            # نسخ المحادثات
+            os.makedirs(f"{backup_dir}/conversations", exist_ok=True)
+            for conv_id, conv_data in self.conversations.items():
+                with open(f"{backup_dir}/conversations/{conv_id}.json", "w", encoding="utf-8") as f:
+                    json.dump(conv_data, f, ensure_ascii=False, indent=2)
+            
+            # نسخ المشاريع
+            with open(f"{backup_dir}/projects.json", "w", encoding="utf-8") as f:
+                json.dump(self.projects, f, ensure_ascii=False, indent=2)
+            
+            # نسخ الأكواد
+            with open(f"{backup_dir}/code_repository.json", "w", encoding="utf-8") as f:
+                json.dump(self.code_repository, f, ensure_ascii=False, indent=2)
+            
+            return backup_dir
+        except Exception as e:
+            logger.error(f"Error during backup: {str(e)}")
+            return None
 
 memory = AIMemory()
 
@@ -247,7 +281,8 @@ class AnalyticsEngine:
                 "positive": result["label"] == "POSITIVE",
                 "negative": result["label"] == "NEGATIVE"
             }
-        except:
+        except Exception as e:
+            logger.warning(f"Sentiment analysis failed, using fallback: {str(e)}")
             # Fallback basic sentiment analysis
             positive_words = ["good", "great", "excellent", "happy", "جيد", "رائع", "ممتاز", "سعيد"]
             negative_words = ["bad", "terrible", "awful", "sad", "سيء", "فظيع", "مزعج", "حزين"]
@@ -275,7 +310,8 @@ class AnalyticsEngine:
         try:
             vectors = self.tfidf.fit_transform([prompt, response])
             relevance_score = cosine_similarity(vectors[0:1], vectors[1:2])[0][0]
-        except:
+        except Exception as e:
+            logger.warning(f"TF-IDF analysis failed: {str(e)}")
             relevance_score = 0.7  # قيمة افتراضية في حالة الخطأ
             
         # تحليل المشاعر
@@ -297,16 +333,33 @@ class AIEngine:
         self.executor = ThreadPoolExecutor(max_workers=8)
         self.models = {}
         self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.quantization_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.float16,
+            bnb_4bit_quant_type="nf4"
+        )
         
-    async def load_model(self, model_type: str, model_name: str = None):
+    async def load_model(self, model_type: str, model_name: str = None, use_large: bool = False):
         """تحميل نموذج معين"""
-        if model_type not in self.models:
+        model_key = f"{model_type}-large" if use_large else model_type
+        
+        if model_key not in self.models:
             try:
                 if model_type == "text":
-                    model_name = model_name or LANGUAGE_MODELS.get("en")
+                    model_name = model_name or (LANGUAGE_MODELS.get(f"{model_type}-large") if use_large else LANGUAGE_MODELS.get("en"))
+                    
+                    if use_large:
+                        model = AutoModelForCausalLM.from_pretrained(
+                            model_name,
+                            quantization_config=self.quantization_config,
+                            device_map="auto",
+                            torch_dtype=torch.float16
+                        )
+                    else:
+                        model = AutoModelForCausalLM.from_pretrained(model_name).to(self.device)
+                    
                     tokenizer = AutoTokenizer.from_pretrained(model_name)
-                    model = AutoModelForCausalLM.from_pretrained(model_name).to(self.device)
-                    self.models[model_type] = {"tokenizer": tokenizer, "model": model}
+                    self.models[model_key] = {"tokenizer": tokenizer, "model": model}
                 
                 elif model_type == "image":
                     scheduler = EulerDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-2", subfolder="scheduler")
@@ -315,36 +368,50 @@ class AIEngine:
                         scheduler=scheduler,
                         torch_dtype=torch.float16
                     ).to(self.device)
-                    self.models[model_type] = model
+                    self.models[model_key] = model
                 
                 elif model_type == "code":
-                    tokenizer = AutoTokenizer.from_pretrained(LANGUAGE_MODELS["code"])
-                    model = AutoModelForCausalLM.from_pretrained(LANGUAGE_MODELS["code"]).to(self.device)
-                    self.models[model_type] = {"tokenizer": tokenizer, "model": model}
+                    if use_large:
+                        model = AutoModelForCausalLM.from_pretrained(
+                            LANGUAGE_MODELS["code-large"],
+                            quantization_config=self.quantization_config,
+                            device_map="auto",
+                            torch_dtype=torch.float16
+                        )
+                    else:
+                        model = AutoModelForCausalLM.from_pretrained(LANGUAGE_MODELS["code"]).to(self.device)
+                    
+                    tokenizer = AutoTokenizer.from_pretrained(LANGUAGE_MODELS["code-large"] if use_large else LANGUAGE_MODELS["code"])
+                    self.models[model_key] = {"tokenizer": tokenizer, "model": model}
                 
-                logger.info(f"تم تحميل النموذج بنجاح: {model_type}")
+                logger.info(f"تم تحميل النموذج بنجاح: {model_key}")
             except Exception as e:
                 logger.error(f"خطأ في تحميل النموذج: {str(e)}")
                 raise
         
-        return self.models[model_type]
+        return self.models[model_key]
     
-    async def generate_text(self, prompt: str, lang: str = None, max_length: int = 300) -> str:
+    async def generate_text(self, prompt: str, lang: str = None, max_length: int = 300, use_large: bool = False) -> str:
         """توليد نص بناء على المطالبة"""
         if not lang:
-            lang = detect(prompt)
+            try:
+                lang = detect(prompt)
+            except:
+                lang = "en"
+        
+        model_name = LANGUAGE_MODELS.get(f"{lang}-large" if use_large else lang, 
+                                      LANGUAGE_MODELS.get("en-large" if use_large else "en"))
         
-        model_name = LANGUAGE_MODELS.get(lang, LANGUAGE_MODELS["en"])
-        model = await self.load_model("text", model_name)
+        model = await self.load_model("text", model_name, use_large)
         
         inputs = model["tokenizer"](prompt, return_tensors="pt").to(self.device)
         outputs = model["model"].generate(**inputs, max_length=max_length, do_sample=True, top_k=50, top_p=0.95)
         
         return model["tokenizer"].decode(outputs[0], skip_special_tokens=True)
     
-    async def generate_code(self, prompt: str, language: str = "python", max_length: int = 500) -> str:
+    async def generate_code(self, prompt: str, language: str = "python", max_length: int = 500, use_large: bool = False) -> str:
         """توليد كود برمجي"""
-        model = await self.load_model("code")
+        model = await self.load_model("code", use_large=use_large)
         
         prompt = f"# Language: {language}\n# Description: {prompt}\n# Code:\n"
         inputs = model["tokenizer"](prompt, return_tensors="pt").to(self.device)
@@ -357,7 +424,10 @@ class AIEngine:
             code=generated_code,
             language=language,
             purpose=prompt[:100],
-            metadata={"generated_at": str(datetime.now())}
+            metadata={
+                "generated_at": str(datetime.now()),
+                "model_used": "large" if use_large else "base"
+            }
         )
         
         return generated_code
@@ -378,8 +448,8 @@ class AIEngine:
         """توليد فيديو من النص (محاكاة)"""
         save_path = f"uploads/generated_video_{int(time.time())}.mp4"
         
-        # إنشاء فيديو مع نص
-        clip = mp.VideoFileClip("assets/blank_video.mp4").set_duration(duration)
+        # إنشاء فيديو مع نص (استخدام صورة سوداء كخلفية)
+        clip = mp.ColorClip(size=(640, 480), color=(0, 0, 0), duration=duration)
         txt_clip = mp.TextClip(prompt, fontsize=24, color='white', size=clip.size).set_position('center').set_duration(duration)
         video = mp.CompositeVideoClip([clip, txt_clip])
         video.write_videofile(save_path, fps=fps)
@@ -512,7 +582,11 @@ class ThinkingEngine:
     
     async def generate_plan(self, prompt: str, task_type: str = "text") -> dict:
         """إنشاء خطة تنفيذية للمهمة"""
-        lang = detect(prompt)
+        try:
+            lang = detect(prompt)
+        except:
+            lang = "en"
+            
         steps = self.get_thinking_steps(task_type, lang)
         
         plan = {
@@ -536,10 +610,12 @@ class GenerationRequest(BaseModel):
     language: Optional[str] = None
     conversation_id: Optional[str] = None
     improvements: Optional[List[str]] = None
+    use_large_model: bool = False  # إضافة خيار استخدام النماذج الكبيرة
 
 class ConversationRequest(BaseModel):
     initial_prompt: str
     project_id: Optional[str] = None
+    use_large_model: bool = False
 
 class ProjectRequest(BaseModel):
     name: str
@@ -550,6 +626,7 @@ class CodeImprovementRequest(BaseModel):
     code: str
     language: str
     improvements: List[str] = Field(..., example=["add_comments", "optimize", "add_error_handling"])
+    use_large_model: bool = False
 
 # 8. نظام إدارة المحادثات عبر WebSocket
 class ConnectionManager:
@@ -585,7 +662,11 @@ async def start_conversation(request: ConversationRequest):
         conv_id=conv_id,
         role="user",
         content=request.initial_prompt,
-        metadata={"type": "text", "project_id": request.project_id}
+        metadata={
+            "type": "text", 
+            "project_id": request.project_id,
+            "use_large_model": request.use_large_model
+        }
     )
     
     return {"conversation_id": conv_id, "name": memory.conversations[conv_id]["name"]}
@@ -606,7 +687,10 @@ async def websocket_conversation(websocket: WebSocket, conversation_id: str):
                     conv_id=conversation_id,
                     role="user",
                     content=message["content"],
-                    metadata={"type": message.get("content_type", "text")}
+                    metadata={
+                        "type": message.get("content_type", "text"),
+                        "use_large_model": message.get("use_large_model", False)
+                    }
                 )
                 
                 # إنشاء خطة للرد
@@ -622,10 +706,19 @@ async def websocket_conversation(websocket: WebSocket, conversation_id: str):
                     await asyncio.sleep(1)
                 
                 # توليد الرد
+                use_large = message.get("use_large_model", False)
+                
                 if content_type == "text":
-                    response = await engine.generate_text(message["content"])
+                    response = await engine.generate_text(
+                        message["content"],
+                        use_large=use_large
+                    )
                 elif content_type == "code":
-                    response = await engine.generate_code(message["content"], message.get("language", "python"))
+                    response = await engine.generate_code(
+                        message["content"],
+                        message.get("language", "python"),
+                        use_large=use_large
+                    )
                 elif content_type == "image":
                     image_path = await engine.generate_image(message["content"])
                     response = f"IMAGE_GENERATED:{image_path}"
@@ -646,7 +739,8 @@ async def websocket_conversation(websocket: WebSocket, conversation_id: str):
                     metadata={
                         "type": content_type,
                         "evaluation": evaluation,
-                        "plan": plan
+                        "plan": plan,
+                        "model_used": "large" if use_large else "base"
                     }
                 )
                 
@@ -698,7 +792,8 @@ async def improve_code(request: CodeImprovementRequest):
         metadata={
             "original_code": request.code,
             "improvements": request.improvements,
-            "analyzed_at": str(datetime.now())
+            "analyzed_at": str(datetime.now()),
+            "model_used": "large" if request.use_large_model else "base"
         }
     )
     
@@ -739,7 +834,8 @@ async def backup_scheduler():
         await asyncio.sleep(3600)  # كل ساعة
         try:
             backup_dir = memory.backup_data()
-            logger.info(f"تم إنشاء نسخة احتياطية في: {backup_dir}")
+            if backup_dir:
+                logger.info(f"تم إنشاء نسخة احتياطية في: {backup_dir}")
         except Exception as e:
             logger.error(f"فشل النسخ الاحتياطي: {str(e)}")
 
@@ -773,7 +869,16 @@ async def startup_event():
     
     logger.info("تم بدء تشغيل MarkAI بنجاح")
 
-# 13. تشغيل التطبيق
+# 13. ملفات إضافية لتهيئة Hugging Face Spaces
+@app.get("/app")
+async def serve_app():
+    return FileResponse("static/index.html")
+
+@app.get("/favicon.ico")
+async def favicon():
+    return FileResponse("static/favicon.ico")
+
+# 14. تشغيل التطبيق
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(app, host="0.0.0.0", port=7860, reload=True)