Deploy RAQIM 2026-05-02 21:13

artifacts/api-server/src/routes/convert.ts

@@ -6,6 +6,7 @@ import { db } from "@workspace/db";
 import { filesTable, conversionsTable } from "@workspace/db";
 import { eq, and } from "drizzle-orm";
 import { requireAuth, AuthRequest } from "../middlewares/auth.js";
+import { logger } from "../lib/logger.js";
 
 const router = Router();
 router.use(requireAuth);
@@ -119,7 +120,13 @@ async function runConversionCore(conversionId: string, fileId: string, storagePa
       rawText = fs.readFileSync(storagePath, "utf-8");
     } else if (ext === ".pdf") {
       rawText = await extractPdf(storagePath, pageStart, pageEnd);
-      await updateProgress("ocr", 38, steps, "تم تحليل الـ PDF واستخراج النص...");
+      await updateProgress("ocr", 32, steps, "تم استخراج النص الخام من الـ PDF...");
+      // AI-powered Arabic text correction — fixes broken font CMap character
+      // transpositions that pdfjs-dist cannot resolve algorithmically.
+      rawText = await correctArabicPdfText(rawText, (msg, pct) =>
+        updateProgress("ocr", pct, steps, msg)
+      );
+      await updateProgress("ocr", 55, steps, "اكتمل التصحيح الذكي للنص العربي ✓");
     } else if ([".docx", ".doc"].includes(ext)) {
       rawText = await extractDocx(storagePath);
       await updateProgress("ocr", 38, steps, "تم استخراج نص ملف Word...");
@@ -634,6 +641,195 @@ function cleanArabicPdfRaw(text: string): string {
   return out.join("\n").replace(/\n{3,}/g, "\n\n").trim();
 }
 
+// ── Arabic PDF AI correction ──────────────────────────────────────────────────
+// Many Arabic PDFs have broken ToUnicode CMaps in their fonts — the glyph→Unicode
+// mapping is wrong, so pdfjs-dist returns correct Unicode codepoints but in the
+// wrong character identity (e.g. ل and م swapped). This cannot be fixed via X-sort
+// or bidi algorithms since the characters themselves are wrong, not their order.
+//
+// Solution: use the Replit AI Integration proxy (OpenAI-compatible, no extra API key
+// needed — auto-provisioned via AI_INTEGRATIONS_OPENAI_BASE_URL) to run GPT-4.1,
+// which has excellent Arabic language support. Falls back to raw text gracefully.
+//
+// Models tried in order via Replit AI Integration proxy (OpenAI-compatible, free via Replit):
+const AI_CORRECTION_MODELS = [
+  "gpt-4.1",       // Best Arabic quality
+  "gpt-4.1-mini",  // Faster fallback
+];
+const MAX_AI_CHUNKS = 30;      // cap to avoid excessively long waits
+const AI_CHUNK_CHARS = 1600;   // chars per chunk (leaves room for system prompt)
+const AI_CHUNK_TIMEOUT_MS = 90_000;
+
+// Split at paragraph boundaries so the LLM gets coherent paragraphs
+function chunkForAiCorrection(text: string): string[] {
+  const paras = text.split(/\n{2,}/);
+  const chunks: string[] = [];
+  let buf = "";
+  for (const para of paras) {
+    const joined = buf ? buf + "\n\n" + para : para;
+    if (joined.length <= AI_CHUNK_CHARS) {
+      buf = joined;
+    } else {
+      if (buf) chunks.push(buf);
+      if (para.length > AI_CHUNK_CHARS) {
+        // Long single paragraph: split by newline
+        for (const line of para.split("\n")) {
+          const lj = buf ? buf + "\n" + line : line;
+          if (lj.length <= AI_CHUNK_CHARS) { buf = lj; }
+          else { if (buf) chunks.push(buf); buf = line.slice(0, AI_CHUNK_CHARS); }
+        }
+      } else {
+        buf = para;
+      }
+    }
+  }
+  if (buf.trim()) chunks.push(buf);
+  return chunks.filter(c => c.trim().length > 0);
+}
+
+// Resolve the best available OpenAI-compatible endpoint.
+// Priority:
+//   1. Replit AI Integration proxy (auto-provisioned on Replit, free)
+//   2. Custom endpoint: OPENAI_BASE_URL + OPENAI_API_KEY (user-supplied in HF Space secrets)
+//   3. None → skip AI correction
+function resolveAiEndpoint(): { baseUrl: string; apiKey: string } | null {
+  const replitUrl = process.env.AI_INTEGRATIONS_OPENAI_BASE_URL;
+  if (replitUrl) {
+    return {
+      baseUrl: replitUrl,
+      apiKey: process.env.AI_INTEGRATIONS_OPENAI_API_KEY ?? "placeholder",
+    };
+  }
+  const customUrl = process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1";
+  const customKey = process.env.OPENAI_API_KEY;
+  if (customKey) {
+    return { baseUrl: customUrl, apiKey: customKey };
+  }
+  return null;
+}
+
+async function callAiCorrection(text: string, model: string): Promise<string> {
+  const endpoint = resolveAiEndpoint();
+  if (!endpoint) throw new Error("no_ai_proxy");
+  const { baseUrl, apiKey } = endpoint;
+
+  const systemMsg =
+    "أنت نظام متخصص في تصحيح نصوص PDF العربية المستخرجة آلياً. " +
+    "النص مصدره كتب إسلامية وأكاديمية عربية. " +
+    "بعض حروف النص قد تبدلت أو انقلبت بسبب خلل في جدول ترميز خط الـ PDF (ToUnicode CMap). " +
+    "مثال على الخلل: 'امحلد هلل' الصحيحة 'الحمد لله'، و'اثنياا' الصح��حة 'ثانياً'، و'عرشة' الصحيحة 'عشرة'، " +
+    "و'عىل' الصحيحة 'على'، و'اذلي' الصحيحة 'الذي'، و'اليت' الصحيحة 'التي'، " +
+    "و'فيرسان' الصحيحة 'فبإمكاننا' أو ما شابه حسب السياق. " +
+    "مهمتك: تصحيح الكلمات المعطوبة فقط مع الحفاظ التام على: " +
+    "١) المعنى والمحتوى ٢) التنسيق (أسطر، فقرات، علامات Markdown) ٣) الكلمات الصحيحة كما هي. " +
+    "لا تضف ولا تحذف محتوى. أعد النص المصحح فقط بدون أي شرح.";
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), AI_CHUNK_TIMEOUT_MS);
+
+  try {
+    const resp = await fetch(`${baseUrl}/chat/completions`, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        model,
+        messages: [
+          { role: "system", content: systemMsg },
+          { role: "user",   content: `النص المستخرج من PDF:\n\n${text}\n\nالنص المصحح:` },
+        ],
+        max_completion_tokens: Math.min(3000, Math.ceil(text.length * 2)),
+      }),
+      signal: controller.signal,
+    });
+
+    if (resp.status === 429) throw Object.assign(new Error("rate_limited"), { code: "rate_limited" });
+    if (!resp.ok) throw new Error(`ai_http_${resp.status}`);
+
+    const data = await resp.json() as any;
+    const corrected = (data.choices?.[0]?.message?.content ?? "").trim();
+
+    // Sanity: corrected must be at least 40% the length of input (not truncated)
+    // and no more than 300% (not hallucinated).
+    if (!corrected || corrected.length < text.length * 0.4 || corrected.length > text.length * 3) {
+      return text;
+    }
+    return corrected;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+type ProgressFn = (msg: string, pct: number) => Promise<void>;
+
+async function correctArabicPdfText(rawText: string, onProgress?: ProgressFn): Promise<string> {
+  if (!resolveAiEndpoint()) {
+    logger.info("[arabic-ai] No AI endpoint configured — skipping AI correction (set OPENAI_API_KEY in HF Space secrets to enable in production)");
+    return rawText;
+  }
+
+  // Only correct if text is predominantly Arabic
+  const arabicChars = (rawText.match(/[\u0600-\u06FF]/g) ?? []).length;
+  const nonSpaceChars = rawText.replace(/\s/g, "").length;
+  if (nonSpaceChars < 50 || arabicChars / nonSpaceChars < 0.25) {
+    return rawText;
+  }
+
+  const chunks = chunkForAiCorrection(rawText);
+  const total = Math.min(chunks.length, MAX_AI_CHUNKS);
+  logger.info(`[arabic-ai] Correcting ${total}/${chunks.length} chunks with GPT`);
+
+  const correctedParts: string[] = [];
+  let modelIdx = 0;
+
+  for (let i = 0; i < chunks.length; i++) {
+    // Uncapped chunks pass through raw (document too large)
+    if (i >= MAX_AI_CHUNKS) {
+      correctedParts.push(...chunks.slice(i));
+      break;
+    }
+
+    // Report progress: map chunk index to 33%–54% range
+    const pct = 33 + Math.round((i / total) * 21);
+    await onProgress?.(`تصحيح النص بالذكاء الاصطناعي... (${i + 1}/${total})`, pct);
+
+    let done = false;
+    let attempts = 0;
+    while (!done && modelIdx < AI_CORRECTION_MODELS.length) {
+      try {
+        const model = AI_CORRECTION_MODELS[modelIdx];
+        const fixed = await callAiCorrection(chunks[i], model);
+        correctedParts.push(fixed);
+        done = true;
+      } catch (err: any) {
+        attempts++;
+        const code = err?.code ?? err?.message ?? "";
+        if (code === "rate_limited") {
+          logger.warn(`[arabic-ai] Rate-limited on ${AI_CORRECTION_MODELS[modelIdx]}, switching model`);
+          modelIdx++;
+        } else if (code === "no_ai_proxy") {
+          logger.warn("[arabic-ai] No AI proxy — using raw text for all remaining chunks");
+          correctedParts.push(...chunks.slice(i));
+          return correctedParts.join("\n\n");
+        } else {
+          logger.warn({ err }, `[arabic-ai] Error on chunk ${i}, trying next model`);
+          modelIdx++;
+        }
+        if (attempts > 3) break;
+      }
+    }
+
+    if (!done) {
+      correctedParts.push(chunks[i]);
+      modelIdx = 0; // reset; models might free up for next chunk
+    }
+  }
+
+  return correctedParts.join("\n\n");
+}
+
 // ── RTL-aware PDF extractor using pdfjs-dist directly ────────────────────────
 // pdf-parse v2 has no `pagerender` callback, so we bypass it and use
 // pdfjs-dist (already installed as pdf-parse's peer) directly.

replit.md

@@ -123,6 +123,21 @@ All timestamps stored as `INTEGER` (milliseconds since epoch). Enums stored as `
 - `GET /api/admin/trash` — list all trashed items
 - `DELETE /api/admin/trash/empty` — empty trash
 
+## Arabic PDF AI Correction
+
+For Arabic PDFs with broken ToUnicode CMap fonts (pdfjs-dist returns wrong characters, e.g. `امحلد هلل` instead of `الحمد لله`), a post-extraction AI correction step is applied.
+
+**How it works**: After pdfjs-dist text extraction, if the text is ≥25% Arabic, it's split into ~1600-char chunks and each chunk is corrected by GPT-4.1 (max 30 chunks = ~48,000 chars). The system prompt explains the CMap corruption with examples, enabling the model to fix character transpositions without changing content.
+
+**Endpoint priority** (checked in order):
+1. `AI_INTEGRATIONS_OPENAI_BASE_URL` — Replit AI Integration proxy (auto-provisioned on Replit, no API key needed)
+2. `OPENAI_BASE_URL` + `OPENAI_API_KEY` — custom OpenAI-compatible endpoint (set in HF Space secrets for production)
+3. Falls back gracefully to uncorrected text if neither is available
+
+**Models tried in order**: `gpt-4.1` → `gpt-4.1-mini` (with model fallback on 429 rate limit)
+
+**To enable on HF Spaces**: Add `OPENAI_API_KEY` (and optionally `OPENAI_BASE_URL` for non-OpenAI endpoints like Together AI or Groq) to the Space secrets.
+
 ## Architect Engine — 100% Free, No External APIs, No Limits
 
 The "Super Architect" is a fully deterministic rule-based engine (`runRuleBasedArchitect` in `convert.ts`).