app.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Arabic & English Golden Engine V2 - Pro Ultra (Parallel, AI Contextual OCR Correction, Tashkeel, Excel Audit)
"""

import os
import re
import time
import logging
import tempfile
import shutil
import json
import threading
import gradio as gr
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, as_completed
from huggingface_hub import HfApi, list_repo_files, login, hf_hub_download

# ============================================================
# 0) الإعدادات والمسارات
# ============================================================
REPO_ID = "Asem75/aiocr_asistant"
SRC_DIR = "process_files"
FINAL_DIR = "Final_Arabic_Files"
EXCEL_DIR = "editor_sync"
SYSTEM_STATUS_PATH = "system/status.json"

HF_TOKEN = os.environ.get("AIOCR_KEY")
if not HF_TOKEN:
    raise ValueError("AIOCR_KEY غير موجود في متغيرات البيئة!")

HF_TOKEN = HF_TOKEN.strip().split()[0]
login(token=HF_TOKEN)
api = HfApi(token=HF_TOKEN)

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Golden_Engine_V2")

# ============================================================
# 1) دوال التحميل البطيء للنماذج (تحديث للنموذج السياقي العملاق الشامل)
# ============================================================
_corrector = None
_tashkeel_model = None

def get_ai_corrector():
    global _corrector
    if _corrector is None:
        from transformers import pipeline
        # استخدام mBART-50 لفهم سياق الجملة بالكامل ومنع رش النقاط العشوائية
        model_name = "facebook/mbart-large-50"
        logger.info("📥 تحميل نموذج mBART الفائق للفهم السياقي الموحد (عربي + إنجليزي)...")
        _corrector = pipeline("text2text-generation", model=model_name)
    return _corrector

def get_tashkeel_model():
    global _tashkeel_model
    if _tashkeel_model is None:
        from transformers import pipeline
        model_name = "bakrianoo/tashkeela-model"
        logger.info("📥 تحميل نموذج التشكيل...")
        _tashkeel_model = pipeline("text2text-generation", model=model_name)
    return _tashkeel_model

# ============================================================
# 2) دالة الترميم الشاملة وسياق النص (تم دمج التقطيع الذكي لمنع انقطاع النص)
# ============================================================
def ai_restore_and_punctuate(text, apply_tashkeel=False):
    try:
        corrector = get_ai_corrector()
        
        # تقطيع النص بناءً على السطور وعلامات الوقف المتاحة لحماية السياق المعنوي والجمل
        sentences = re.split(r'(?<=[.!?؟،;\n])\s+', text)
        corrected_paragraphs = []
        
        current_chunk = ""
        for sentence in sentences:
            # تجميع النص في كتل آمنة (حوالي 400 حرف) لضمان الفهم السياقي الكامل دون انقطاع
            if len(current_chunk) + len(sentence) < 400:
                current_chunk += sentence + " "
            else:
                if current_chunk.strip():
                    res = corrector(current_chunk.strip(), max_length=512, clean_up_tokenization_spaces=True)
                    corrected_paragraphs.append(res[0]['generated_text'])
                current_chunk = sentence + " "
                
        # معالجة ما تبقى من النص
        if current_chunk.strip():
            res = corrector(current_chunk.strip(), max_length=512, clean_up_tokenization_spaces=True)
            corrected_paragraphs.append(res[0]['generated_text'])

        restored = "\n".join(corrected_paragraphs)

        if apply_tashkeel:
            try:
                tashkeel = get_tashkeel_model()
                paragraphs = restored.split('\n')
                tashkeel_paragraphs = []
                for para in paragraphs:
                    if para.strip():
                        res = tashkeel(para, max_length=512)
                        tashkeel_paragraphs.append(res[0]['generated_text'])
                    else:
                        tashkeel_paragraphs.append("")
                restored = "\n".join(tashkeel_paragraphs)
            except Exception as e:
                logger.warning(f"فشل التشكيل: {e}")
        return restored
    except Exception as e:
        logger.warning(f"فشل نموذج التصحيح السياقي، اللجوء للقواعد: {e}")
        return fallback_deterministic_clean(text)

def fallback_deterministic_clean(text):
    from pyarabic.araby import strip_tashkeel, normalize_ligature
    COMMON = set("الله هذا ذلك لكن لأن فإن كان تكون الذين الذي التي".split())
    lines = text.split('\n')
    fixed = []
    for line in lines:
        words = line.split()
        new_words = []
        for w in words:
            base = strip_tashkeel(normalize_ligature(w))
            if base in COMMON:
                new_words.append(base)
            else:
                base = base.replace("اللا", "لا").replace("ى", "ي").replace("ة", "ه")
                new_words.append(base)
        fixed.append(" ".join(new_words))
    return "\n".join(fixed)

# ============================================================
# 3) مقارنة النصوص
# ============================================================
def compare_texts(original, corrected):
    orig_words = original.split()
    corr_words = corrected.split()
    result = []
    for i in range(max(len(orig_words), len(corr_words))):
        if i < len(orig_words) and i < len(corr_words):
            ow = orig_words[i]
            cw = corr_words[i]
            if ow != cw:
                if len(set(ow) - set(cw)) > 2 or len(set(cw) - set(ow)) > 2:
                    status = "doubtful"
                else:
                    status = "corrected"
                result.append((ow, cw, status))
            else:
                result.append((ow, cw, "unchanged"))
        elif i < len(orig_words):
            result.append((orig_words[i], "", "removed"))
        else:
            result.append(("", corr_words[i], "added"))
    return result

# ============================================================
# 4) إنشاء ملف الإكسيل المحسّن
# ============================================================
def generate_excel_advanced(original_text, corrected_text):
    import openpyxl
    from openpyxl.styles import PatternFill, Font

    wb = openpyxl.Workbook()
    ws1 = wb.active
    ws1.title = "النص المعالج"

    sentences = re.split(r'([.،؟!؛:])', corrected_text)
    merged = []
    temp = ""
    for part in sentences:
        if part in ['.', '،', '؟', '!', '؛', ':']:
            temp += part
            merged.append(temp.strip())
            temp = ""
        else:
            if temp:
                merged.append(temp.strip())
            temp = part
    if temp:
        merged.append(temp.strip())

    white_fill = PatternFill(start_color="FFFFFF", end_color="FFFFFF", fill_type="solid")
    yellow_fill = PatternFill(start_color="FFFF00", end_color="FFFF00", fill_type="solid")
    red_fill = PatternFill(start_color="FF0000", end_color="FF0000", fill_type="solid")

    comparisons = compare_texts(original_text, corrected_text)
    word_status = {}
    for orig, corr, status in comparisons:
        if orig:
            word_status[orig] = (corr, status)

    row = 1
    for sentence in merged:
        words = sentence.split()
        if not words:
            continue
        for col, word in enumerate(words, 1):
            cell = ws1.cell(row=row, column=col, value=word)
            status = None
            for key, (corr_val, st) in word_status.items():
                if key == word or corr_val == word:
                    status = st
                    break
            if status == "unchanged":
                cell.fill = white_fill
            elif status == "corrected":
                cell.fill = yellow_fill
            elif status == "doubtful":
                cell.fill = red_fill
            else:
                cell.fill = white_fill
        row += 1

    ws2 = wb.create_sheet("تقرير التصحيحات")
    ws2.append(["الكلمة الأصلية", "الكلمة المصححة", "الحالة"])
    for orig, corr, status in comparisons:
        ws2.append([orig, corr, status])
    for cell in ws2[1]:
        cell.font = Font(bold=True)

    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".xlsx")
    wb.save(tmp.name)
    return tmp.name

# ============================================================
# 5) معالجة ملف واحد
# ============================================================
def process_single_file(remote_file_path):
    try:
        filename = os.path.basename(remote_file_path)
        logger.info(f"بدء معالجة: {filename}")

        clean_name = os.path.splitext(filename)[0]
        apply_tashkeel = False
        if clean_name.startswith("tash_"):
            apply_tashkeel = True

        local_path = hf_hub_download(
            repo_id=REPO_ID,
            filename=remote_file_path,
            repo_type="dataset",
            token=HF_TOKEN
        )

        with open(local_path, "r", encoding="utf-8") as f:
            raw_text = f.read()

        try:
            api.delete_file(path_in_repo=remote_file_path, repo_id=REPO_ID, repo_type="dataset", token=HF_TOKEN)
        except Exception as e:
            logger.warning(f"لم يتم حذف الملف من process: {e}")

        corrected = ai_restore_and_punctuate(raw_text, apply_tashkeel=apply_tashkeel)
        excel_path = generate_excel_advanced(raw_text, corrected)

        final_txt_name = f"{clean_name}.txt"
        final_txt_path = f"/tmp/{final_txt_name}"
        with open(final_txt_path, "w", encoding="utf-8") as f:
            f.write(corrected)
        api.upload_file(
            path_or_fileobj=final_txt_path,
            path_in_repo=f"{FINAL_DIR}/{final_txt_name}",
            repo_id=REPO_ID, repo_type="dataset", token=HF_TOKEN
        )

        excel_final_name = f"{clean_name}.xlsx"
        api.upload_file(
            path_or_fileobj=excel_path,
            path_in_repo=f"{EXCEL_DIR}/{excel_final_name}",
            repo_id=REPO_ID, repo_type="dataset", token=HF_TOKEN
        )

        os.remove(local_path)
        os.remove(final_txt_path)
        os.remove(excel_path)

        update_system_status(f"✅ تمت معالجة {filename} بنجاح")
        logger.info(f"✔ انتهت معالجة {filename}")

    except Exception as e:
        logger.error(f"فشل في معالجة الملف {remote_file_path}: {e}")

# ============================================================
# 6) الرادار المتوازي
# ============================================================
def monitor_and_execute():
    logger.info("🚀 انطلاق المعالج الذهبي V2 مع المعالجة المتوازية...")
    update_system_status("المعالج الذهبي V2 يعمل - مراقبة process_files")

    with ThreadPoolExecutor(max_workers=3) as executor:
        while True:
            try:
                all_files = list_repo_files(REPO_ID, repo_type="dataset")
                process_files = [f for f in all_files if f.startswith(f"{SRC_DIR}/") and f.endswith(".txt")]

                if not process_files:
                    time.sleep(5)
                    continue

                vip = [f for f in process_files if os.path.basename(f).startswith("VIP_")]
                free = [f for f in process_files if not os.path.basename(f).startswith("VIP_")]
                ordered = vip + free

                futures = {executor.submit(process_single_file, f): f for f in ordered}
                for future in as_completed(futures):
                    pass

            except Exception as e:
                logger.error(f"خطأ في الرادار: {e}")
                time.sleep(10)

# ============================================================
# 7) تحديث status.json
# ============================================================
def update_system_status(message):
    local_path = "/tmp/status.json"
    data = {"status": message, "updated": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
    with open(local_path, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=4)
    api.upload_file(
        path_or_fileobj=local_path,
        path_in_repo=SYSTEM_STATUS_PATH,
        repo_id=REPO_ID, repo_type="dataset", token=HF_TOKEN
    )

# ============================================================
# 8) واجهة Gradio
# ============================================================
def get_status():
    try:
        with open("/tmp/status.json", "r") as f:
            data = json.load(f)
            return f"{data['status']} | {data['updated']}"
    except:
        return "قيد التشغيل..."

with gr.Blocks(title="Golden Engine V2") as demo:
    gr.Markdown("# ⚡ المعالج الذهبي V2 (معالجة متوازية + تصحيح سياقي متقدم عربي وإنجليزي)")
    gr.Markdown("يراقب `process_files` ويعالج بالتوازي مع حماية كاملة للسياق والنصوص الطويلة من الانقطاع")
    status_text = gr.Textbox(label="الحالة", value="...", interactive=False)
    timer = gr.Timer(3)
    timer.tick(fn=get_status, outputs=status_text)

# تشغيل الرادار في خيط منفصل
threading.Thread(target=monitor_and_execute, daemon=True).start()

demo.launch(server_name="0.0.0.0", server_port=7860)