Update app.py

app.py

@@ -1,100 +1,218 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Arabic Golden Engine – Pro Lite Edition (Stable)
+"""
+
 import os
-import json
 import re
-from datetime import datetime
-from fastapi import FastAPI
-from pydantic import BaseModel
-
-app = FastAPI()
-
-# ============================
-# 1) مجلدات النظام
-# ============================
-BASE_DIR = "Final_Arabic_Files"
-STATUS_FILE = "system/status.json"
-MESSAGES_FILE = "system/messages.json"
-
-os.makedirs(BASE_DIR, exist_ok=True)
-os.makedirs("system", exist_ok=True)
-
-# ============================
-# 2) نموذج استقبال JSON
-# ============================
-class OCRPayload(BaseModel):
-    text: str
-    username: str
-    rank: str | None = None   # ← إصلاح المشكلة هنا
-
-# ============================
-# 3) دوال مساعدة
-# ============================
-def clean_text(text):
-    text = re.sub(r'[^\u0600-\u06FF0-9\s.,!?؛:()\-]', '', text)
-    text = re.sub(r'\s+', ' ', text).strip()
+import queue
+import threading
+import time
+import logging
+import tempfile
+import shutil
+import json
+import pandas as pd
+import gradio as gr
+from huggingface_hub import HfApi
+from fastapi import FastAPI, Request
+from pyarabic.araby import strip_tashkeel, normalize_ligature
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+# ========== إعدادات المستودع ==========
+REPO_ID = "Asem75/aiocr_asistant"
+FINAL_DIR = "Final_Arabic_Files"
+EXCEL_DIR = "editor_sync"
+
+LOCAL_FREE_TXT_DIR = "./free_processed_txt_queue"
+LOCAL_FREE_XLS_DIR = "./free_processed_xls_queue"
+os.makedirs(LOCAL_FREE_TXT_DIR, exist_ok=True)
+os.makedirs(LOCAL_FREE_XLS_DIR, exist_ok=True)
+
+HF_TOKEN = os.environ.get("AIOCR_KEY")
+api = HfApi(token=HF_TOKEN.strip().split()[0]) if HF_TOKEN else None
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("Golden_Engine")
+
+# ========== طابور المعالجة ==========
+ARABIC_ENGINE_QUEUE = queue.Queue()
+latest_processed_text = ""
+
+# ========== نسخة Pro Lite — دوال التصحيح ==========
+COMMON_AR = set("""
+الله هذا ذلك لكن لأن فإن كان تكون الذين الذي التي عندما حيث بينما أيضا جدا فقط بين حتى عليه عليها لديهم لديه منها منه فيها فيه منهم نحن أنت هو هي هم هن ثم بعد قبل خلال بسبب بدون حول تحت فوق ضد عبر نحو عند الى إلى على عن مع من ما ماذا لماذا كيف متى أين هنا هناك نعم لا ربما قد لقد إن أن إذا إذ
+""".split())
+
+PHONETIC_MAP = {
+    "هاذا": "هذا",
+    "هده": "هذه",
+    "دالك": "ذلك",
+    "لأنو": "لأنه",
+    "علشان": "لأن",
+}
+
+VISUAL_MAP = {
+    "اﻟ": "ال",
+    "ى": "ي",
+    "ئ": "ي",
+    "ؤ": "و",
+    "ة": "ه",
+    "اللا": "لا",
+}
+
+def pro_fix_word(word):
+    base = strip_tashkeel(normalize_ligature(word))
+
+    if base in COMMON_AR:
+        return base
+
+    if base in PHONETIC_MAP:
+        return PHONETIC_MAP[base]
+
+    for k, v in VISUAL_MAP.items():
+        if k in base:
+            return base.replace(k, v)
+
+    base2 = re.sub(r"(.)\1{2,}", r"\1\1", base)
+    return base2
+
+def pro_lite_disambiguate(text):
+    words = text.split()
+    fixed = [pro_fix_word(w) for w in words]
+    return " ".join(fixed)
+
+# ========== تنظيف + ترقيم + تنسيق ==========
+def mechanical_clean(text: str) -> str:
+    text = re.sub(r"--- صفحة \d+ \(OCR\) ---", "", text)
+    text = re.sub(r"Page \d+", "", text)
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+def insert_punctuation(text: str) -> str:
+    text = re.sub(r'\b(أولاً|ثانياً|ثالثاً|ملاحظة|تنبيه|مثال)\b\s*', r'\1: ', text)
+    return text
+
+def final_format(text: str) -> str:
+    lines = []
+    for chunk in text.split("\n"):
+        chunk = chunk.strip()
+        if not chunk:
+            continue
+        if not chunk.endswith((".", "،", "؟", "!", "؛", ":")):
+            chunk += "."
+        lines.append(chunk)
+    return "\n\n".join(lines)
+
+def process_ocr_text(raw_text: str) -> str:
+    if not raw_text.strip():
+        return ""
+    text = mechanical_clean(raw_text)
+    text = pro_lite_disambiguate(text)
+    text = insert_punctuation(text)
+    text = final_format(text)
     return text
 
-def save_status(username, filename):
-    status = {
-        "username": username,
-        "filename": filename,
-        "timestamp": datetime.now().isoformat()
-    }
-    with open(STATUS_FILE, "w", encoding="utf-8") as f:
-        json.dump(status, f, ensure_ascii=False, indent=2)
-
-def save_message(username, text):
-    entry = {
-        "username": username,
-        "text": text,
-        "timestamp": datetime.now().isoformat()
-    }
-
-    if not os.path.exists(MESSAGES_FILE):
-        with open(MESSAGES_FILE, "w", encoding="utf-8") as f:
-            json.dump([entry], f, ensure_ascii=False, indent=2)
-        return
-
-    with open(MESSAGES_FILE, "r", encoding="utf-8") as f:
-        data = json.load(f)
-
-    data.append(entry)
-
-    with open(MESSAGES_FILE, "w", encoding="utf-8") as f:
-        json.dump(data, f, ensure_ascii=False, indent=2)
-
-# ============================
-# 4) نقطة استقبال الـ OCR
-# ============================
-@app.post("/receive_ocr")
-def receive_ocr(payload: OCRPayload):
-
-    username = payload.username.strip()
-    text = payload.text.strip()
-
-    # تنظيف النص
-    cleaned = clean_text(text)
-
-    # إنشاء مجلد المستخدم
-    user_dir = os.path.join(BASE_DIR, username)
-    os.makedirs(user_dir, exist_ok=True)
-
-    # اسم الملف النهائي
-    filename = f"{username}_document.txt"
-    filepath = os.path.join(user_dir, filename)
-
-    # حفظ الملف yuh
-    with open(filepath, "w", encoding="utf-8") as f:
-        f.write(cleaned)
-
-    # تحديث status.json
-    save_status(username, filename)
-
-    # تحديث messages.json
-    save_message(username, cleaned)
-
-    return {
-        "status": "success",
-        "username": username,
-        "file": filename,
-        "saved_to": filepath
-    }
+# ========== Excel ==========
+def generate_excel_grid(text: str) -> str:
+    words = [w.strip() for w in text.split() if w.strip()]
+    df = pd.DataFrame(words, columns=["الكلمات المصححة"])
+    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".xlsx")
+    df.to_excel(tmp.name, index=False, engine='openpyxl')
+    return tmp.name
+
+# ========== حلقة المعالجة ==========
+def in_memory_receiver_loop():
+    global latest_processed_text
+    logger.info("🚀 Pro Lite Engine Ready...")
+
+    while True:
+        try:
+            try:
+                raw_text, metadata = ARABIC_ENGINE_QUEUE.get(timeout=1)
+            except queue.Empty:
+                continue
+
+            source = metadata.get("source", "unknown")
+            rank = metadata.get("rank", "free")
+            is_vip = (rank == "vip")
+
+            clean_name = os.path.splitext(os.path.basename(source))[0]
+            txt_filename = f"{clean_name}.txt"
+            xls_filename = f"{clean_name}.xlsx"
+
+            logger.info("📥 معالجة: %s (%s)", txt_filename, rank)
+
+            final_output = process_ocr_text(raw_text)
+            latest_processed_text = final_output
+            excel_path = generate_excel_grid(final_output)
+
+            if api:
+                if is_vip:
+                    with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as tmp_txt:
+                        tmp_txt.write(final_output.encode("utf-8"))
+                        tmp_txt_path = tmp_txt.name
+
+                    api.upload_file(
+                        path_or_fileobj=tmp_txt_path,
+                        path_in_repo=f"{FINAL_DIR}/{txt_filename}",
+                        repo_id=REPO_ID, repo_type="dataset", token=HF_TOKEN
+                    )
+                    os.unlink(tmp_txt_path)
+
+                    api.upload_file(
+                        path_or_fileobj=excel_path,
+                        path_in_repo=f"{EXCEL_DIR}/{xls_filename}",
+                        repo_id=REPO_ID, repo_type="dataset", token=HF_TOKEN
+                    )
+
+                    logger.info("⚡ VIP رفع مباشر: %s", txt_filename)
+
+                else:
+                    shutil.copy2(excel_path, os.path.join(LOCAL_FREE_XLS_DIR, xls_filename))
+                    with open(os.path.join(LOCAL_FREE_TXT_DIR, txt_filename), "w", encoding="utf-8") as f:
+                        f.write(final_output)
+                    logger.info("🐢 FREE تخزين مؤقت: %s", txt_filename)
+
+            if os.path.exists(excel_path):
+                os.unlink(excel_path)
+
+            ARABIC_ENGINE_QUEUE.task_done()
+
+        except Exception as e:
+            logger.error("خطأ: %s", e)
+            time.sleep(1)
+
+# ========== استقبال الـ OCR ==========
+def receive_from_ocr_space(raw_text: str, username: str, rank: str):
+    prefix = "VIP_" if rank == "vip" else ""
+    simulated_source = f"{prefix}{username}_document"
+    ARABIC_ENGINE_QUEUE.put((raw_text, {"source": simulated_source, "rank": rank}))
+    return f"تم استقبال النص من {username}."
+
+# ========== FastAPI ==========
+fastapi_app = FastAPI()
+
+@fastapi_app.post("/receive_ocr")
+async def receive_ocr(request: Request):
+    payload = await request.json()
+    raw_text = payload.get("text", "")
+    username = payload.get("username", "UnknownUser")
+    rank = payload.get("rank", "free")
+    return receive_from_ocr_space(raw_text, username, rank)
+
+# ========== Gradio ==========
+with gr.Blocks(title="Arabic Golden Engine – Pro Lite") as demo:
+    gr.Markdown("# ⚡ المعالج الذهبي – نسخة Pro Lite")
+    gr.Textbox(label="آخر نص", value=lambda: latest_processed_text, every=2)
+
+# ========== تشغيل ==========
+threading.Thread(target=in_memory_receiver_loop, daemon=True).start()
+
+app = gr.mount_gradio_app(fastapi_app, demo, path="/")
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)