Update app.py

app.py

@@ -1,186 +1,149 @@
 import gradio as gr
 from openai import OpenAI
 import os
+import chromadb
+from chromadb.utils import embedding_functions
+import pypdf
+import uuid
+
+# --- الإعدادات الفنية ---
+STORAGE_PATH = "/data/neural_memory" if os.path.exists("/data") else "./neural_memory"
+chroma_client = chromadb.PersistentClient(path=STORAGE_PATH)
+default_ef = embedding_functions.DefaultEmbeddingFunction()
+collection = chroma_client.get_or_create_collection(name="advanced_brain", embedding_function=default_ef)
+
+# --- 1. الابتكار في الحقن (Semantic Ingestion) ---
+def advanced_ingest(file_path):
+    """حقن متقدم مع Metadata و Overlap و Normalization."""
+    try:
+        text = ""
+        filename = os.path.basename(file_path)
+        
+        if file_path.endswith('.pdf'):
+            reader = pypdf.PdfReader(file_path)
+            pages_data = [(p.extract_text(), i+1) for i, p in enumerate(reader.pages)]
+        else:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                pages_data = [(f.read(), 1)]
+
+        documents, metadatas, ids = [], [], []
+        
+        # إعدادات الـ Chunking الاحترافية
+        chunk_size = 1000
+        overlap = 200 # تداخل 20% لضمان اتصال المعنى
+        
+        for content, page_num in pages_data:
+            content = content.replace('\t', ' ').strip() # Normalization بسيط
+            
+            for i in range(0, len(content), chunk_size - overlap):
+                chunk = content[i : i + chunk_size]
+                documents.append(chunk)
+                metadatas.append({
+                    "source": filename,
+                    "page": page_num,
+                    "length": len(chunk)
+                })
+                ids.append(str(uuid.uuid4()))
+        
+        collection.add(documents=documents, metadatas=metadatas, ids=ids)
+        return f"✅ تم حقن {len(documents)} قطعة معرفية من '{filename}' مع حفظ الميتا-داتا."
+    except Exception as e:
+        return f"❌ فشل الحقن: {str(e)}"
+
+# --- 2. الاسترجاع الذكي (Filtered Query) ---
+def smart_query(user_query, threshold=0.6):
+    """استرجاع مع تصفية حسب درجة التشابه (Score Filtering)."""
+    # نطلب نتائج أكثر ثم نصفيها
+    results = collection.query(
+        query_texts=[user_query],
+        n_results=10,
+        include=['documents', 'metadatas', 'distances']
+    )
+    
+    context_parts = []
+    for doc, meta, dist in zip(results['documents'][0], results['metadatas'][0], results['distances'][0]):
+        # في ChromaDB الـ distance الأقل تعني تشابه أكبر (0 = متطابق)
+        # نحولها إلى Score افتراضي (1 - dist)
+        score = 1 - dist
+        if score >= threshold:
+            source_info = f"[المصدر: {meta['source']} | صفحة: {meta['page']}]"
+            context_parts.append(f"{source_info}\n{doc}")
+    
+    return "\n\n---\n\n".join(context_parts) if context_parts else "لم يتم العثور على معرفة وثيقة الصلة."
 
-# ---------------------------------------------------------
-# 1. إعدادات الاتصال والنموذج
-# ---------------------------------------------------------
-hf_token = os.getenv("HF_TOKEN")
-
-client = OpenAI(
-    base_url="https://router.huggingface.co/hf-inference/v1",
-    api_key=hf_token
-)
-
-MODEL_ID = "huihui-ai/Qwen2.5-72B-Instruct-abliterated"
-
-# ---------------------------------------------------------
-# 2. منطق المعالجة (Back-end Logic)
-# ---------------------------------------------------------
-def predict(message, history, system_prompt, temperature, max_tokens):
-    """
-    دالة التفاعل مع النموذج.
-    تستقبل الرسالة، التاريخ، وإعدادات التخصيص.
-    """
-    messages_payload = []
-
-    # إضافة طبقة التخصيص (System Prompt) إذا وجدت
-    if system_prompt and system_prompt.strip():
-        messages_payload.append({"role": "system", "content": system_prompt})
-
-    # تحويل سجل المحادثة من صيغة Gradio إلى صيغة OpenAI
-    # history يأتي كقائمة: [[user_msg, bot_msg], ...]
-    for user_msg, bot_msg in history:
-        if user_msg:
-            messages_payload.append({"role": "user", "content": user_msg})
-        if bot_msg:
-            messages_payload.append({"role": "assistant", "content": bot_msg})
+# --- 3. المحرك العصبي (The Engine) ---
+def neural_engine(message, history, system_prompt, base_url, api_key, temp, score_threshold):
+    client = OpenAI(
+        base_url=base_url or "https://router.huggingface.co/hf-inference/v1",
+        api_key=api_key or os.getenv("HF_TOKEN")
+    )
+    
+    knowledge = smart_query(message, threshold=score_threshold)
     
-    # إضافة الرسالة الحالية
-    messages_payload.append({"role": "user", "content": message})
+    enhanced_system = f"{system_prompt}\n\n[سياق المعرفة الموثق]:\n{knowledge}"
+    
+    messages = [{"role": "system", "content": enhanced_system}]
+    for u, a in history:
+        messages.append({"role": "user", "content": u})
+        messages.append({"role": "assistant", "content": a})
+    messages.append({"role": "user", "content": message})
 
     try:
         response = client.chat.completions.create(
-            model=MODEL_ID,
-            messages=messages_payload,
-            max_tokens=int(max_tokens),
-            temperature=float(temperature),
+            model="huihui-ai/Qwen2.5-72B-Instruct-abliterated",
+            messages=messages,
+            temperature=temp,
             stream=True
         )
-
-        partial_message = ""
+        full_resp = ""
         for chunk in response:
             if chunk.choices[0].delta.content:
-                partial_message += chunk.choices[0].delta.content
-                yield partial_message
-
+                full_resp += chunk.choices[0].delta.content
+                yield full_resp
     except Exception as e:
-        yield f"⚠️ System Error: {str(e)}"
-
-# ---------------------------------------------------------
-# 3. واجهة المستخدم الاحترافية (Front-end / UI)
-# ---------------------------------------------------------
-# استخدام ثيم ناعم واحترافي
-theme = gr.themes.Soft(
-    primary_hue="blue",
-    secondary_hue="slate",
-    neutral_hue="slate",
-)
+        yield f"⚠️ Neural Glitch: {str(e)}"
 
-css = """
-footer {visibility: hidden}
-.container {max-width: 1200px; margin: auto; padding-top: 20px}
-"""
-
-with gr.Blocks(theme=theme, css=css, title="Qwen Pro Interface") as demo:
+# --- 4. واجهة المستخدم المتقدمة ---
+with gr.Blocks(theme=gr.themes.Soft(primary_hue="teal"), css=".gradio-container {background: #f9f9f9}") as demo:
+    gr.Markdown("# 🧬 Neural OS v4.0 (Semantic Edition)")
     
-    gr.Markdown(
-        """
-        # 🤖 Qwen 2.5-72B Professional Interface
-        <div style='opacity: 0.7; font-size: 0.9em;'>
-        Advanced interface for 'huihui-ai/Qwen2.5-72B-Instruct-abliterated' via HF Inference Router.
-        </div>
-        """
-    )
-
     with gr.Tabs():
-        # ================= TAB 1: ساحة المحادثة =================
-        with gr.TabItem("💬 Chat Workspace", id="chat_tab"):
+        with gr.Tab("💬 Interaction Console"):
+            chatbot = gr.Chatbot(height=600, show_label=False)
             with gr.Row():
-                # عمود المحادثة
-                with gr.Column(scale=4):
-                    chatbot = gr.Chatbot(
-                        height=600,
-                        placeholder="Start a new conversation...",
-                        show_copy_button=True,
-                        avatar_images=(None, "🤖"), # يمكن وضع مسار صورة للمستخدم وللبوت
-                        layout="bubble"
-                    )
-                    
-                    with gr.Row():
-                        msg_input = gr.Textbox(
-                            placeholder="Type your message here...",
-                            show_label=False,
-                            scale=8,
-                            container=False,
-                            autofocus=True
-                        )
-                        send_btn = gr.Button("Send ➤", variant="primary", scale=1)
-                        clear_btn = gr.Button("🗑️ Clear", variant="stop", scale=1)
+                msg_input = gr.Textbox(placeholder="اسأل العقل الاصطناعي...", scale=8)
+                submit_btn = gr.Button("نفاذ", variant="primary")
 
-        # ================= TAB 2: إعدادات النموذج =================
-        with gr.TabItem("⚙️ Model Settings & Customization", id="settings_tab"):
-            gr.Markdown("### 🧠 Model Behavior (Custom Layer)")
-            
+        with gr.Tab("📚 Knowledge Vault"):
             with gr.Row():
                 with gr.Column():
-                    system_prompt_input = gr.TextArea(
-                        label="System Prompt (Custom Instruction Layer)",
-                        value="You are a helpful, professional, and intelligent AI assistant.",
-                        placeholder="Define the AI's persona, rules, or role here. (e.g., 'You are a senior python developer...')",
-                        lines=5,
-                        info="This acts as a base layer for the model's behavior throughout the chat."
-                    )
+                    file_input = gr.File(label="وثائق التدريب (PDF/TXT)")
+                    upload_btn = gr.Button("بدء المعالجة الدلالية", variant="secondary")
+                with gr.Column():
+                    status_log = gr.TextArea(label="سجل العمليات", interactive=False)
             
-            gr.Markdown("### 🎛️ Inference Parameters")
+        with gr.Tab("⚙️ Control Panel"):
             with gr.Row():
                 with gr.Column():
-                    temp_slider = gr.Slider(
-                        minimum=0.0, maximum=1.5, value=0.7, step=0.05,
-                        label="Temperature (Creativity)",
-                        info="Lower values for precise facts, higher for creative writing."
-                    )
+                    sys_p = gr.TextArea(label="System Persona", value="أنت محرك معرفي يستند إلى وثائق رسمية.")
+                    score_th = gr.Slider(0.0, 1.0, 0.4, label="Relevance Threshold", info="كلما زاد، كان الاسترجاع أدق وأقل كمية.")
                 with gr.Column():
-                    tokens_slider = gr.Slider(
-                        minimum=256, maximum=8192, value=2048, step=256,
-                        label="Max Tokens (Response Length)",
-                        info="Maximum number of tokens the model can generate."
-                    )
+                    endpoint = gr.Textbox(label="API Endpoint")
+                    token = gr.Textbox(label="Access Token", type="password")
+                    temp = gr.Slider(0, 1.5, 0.7, label="Temperature")
 
-    # ---------------------------------------------------------
-    # 4. ربط المكونات (Wiring)
-    # ---------------------------------------------------------
+    # التفاعلات
+    upload_btn.click(lambda files: "\n".join([advanced_ingest(f.name) for f in files]), [file_input], [status_log])
     
-    # دالة مساعدة لتحديث واجهة المستخدم (مسح صندوق النص وإضافة رسالة المستخدم للشات)
-    def user_turn(user_message, history):
-        return "", history + [[user_message, None]]
-
-    # دالة مساعدة لتوليد الرد وتحديث الشات
-    def bot_turn(history, system_p, temp, tokens):
-        user_message = history[-1][0]
-        # نقوم باستدعاء المولد (generator)
-        bot_response_generator = predict(user_message, history[:-1], system_p, temp, tokens)
-        
-        history[-1][1] = ""
-        for chunk in bot_response_generator:
-            history[-1][1] = chunk
-            yield history
-
-    # عند الضغط على زر الإرسال أو Enter
-    send_event = msg_input.submit(
-        user_turn, 
-        [msg_input, chatbot], 
-        [msg_input, chatbot], 
-        queue=False
-    ).then(
-        bot_turn, 
-        [chatbot, system_prompt_input, temp_slider, tokens_slider], 
-        chatbot
-    )
-
-    click_event = send_btn.click(
-        user_turn, 
-        [msg_input, chatbot], 
-        [msg_input, chatbot], 
-        queue=False
-    ).then(
-        bot_turn, 
-        [chatbot, system_prompt_input, temp_slider, tokens_slider], 
-        chatbot
-    )
-
-    # زر المسح
-    clear_btn.click(lambda: None, None, chatbot, queue=False)
+    def chat_logic(m, h, sp, url, t, tmp, th):
+        gen = neural_engine(m, h, sp, url, t, tmp, th)
+        h.append([m, ""])
+        for res in gen:
+            h[-1][1] = res
+            yield "", h
+
+    submit_btn.click(chat_logic, [msg_input, chatbot, sys_p, endpoint, token, temp, score_th], [msg_input, chatbot])
+    msg_input.submit(chat_logic, [msg_input, chatbot, sys_p, endpoint, token, temp, score_th], [msg_input, chatbot])
 
 if __name__ == "__main__":
     demo.queue().launch()