File size: 2,515 Bytes
325b94c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | from typing import List
from io import BytesIO
import re
from PyPDF2 import PdfReader
from PyPDF2.errors import DependencyError
def extract_text_pypdf2(pdf_bytes: bytes) -> List[str]:
pages: List[str] = []
try:
reader = PdfReader(BytesIO(pdf_bytes))
except DependencyError:
# AES encryption without pycryptodome
return []
# لو PDF مشفر
if reader.is_encrypted:
try:
reader.decrypt("") # حاول password فاضي
except Exception:
return []
for page in reader.pages:
try:
text = page.extract_text() or ""
except Exception:
text = ""
text = re.sub(r"\s+\n", "\n", text)
text = re.sub(r"\n\s+", "\n", text)
text = re.sub(r"[ \t]+", " ", text)
pages.append(text.strip())
return pages
ARABIC_RANGE = re.compile(r"[\u0600-\u06FF]")
LATIN_RANGE = re.compile(r"[A-Za-z]")
def is_text_usable(text: str, min_len: int = 300) -> bool:
if not text or len(text) < min_len:
return False
total_len = len(text)
arabic_chars = len(ARABIC_RANGE.findall(text))
latin_chars = len(LATIN_RANGE.findall(text))
spaces = text.count(" ")
# نسبة حروف مفهومة
readable_ratio = (arabic_chars + latin_chars) / total_len
# -------------------------
# 1️⃣ Check المسافات
# -------------------------
space_ratio = spaces / total_len
if space_ratio < 0.02: # أقل من 2% مسافات = حروف لازقة
return False
# -------------------------
# 2️⃣ تحليل الكلمات
# -------------------------
words = [w for w in text.split() if len(w) > 1]
if len(words) < 20:
return False
avg_word_len = sum(len(w) for w in words) / len(words)
# متوسط كلمة مبالغ فيه
if avg_word_len > 12:
return False
# -------------------------
# 3️⃣ كلمات طويلة جدًا (OCR خداع)
# -------------------------
long_words = [w for w in words if len(w) > 15]
long_ratio = len(long_words) / len(words)
if long_ratio > 0.25: # ربع الكلمات طويلة جدًا
return False
# -------------------------
# 4️⃣ Checks اللغات (زي ما كانت)
# -------------------------
if arabic_chars > 50:
return True
if latin_chars > 100:
return True
if readable_ratio < 0.2:
return False
return True
|