feat: NLP-3 PuncAra-v1 Integration — Local punctuation model - Created src/nlp/punctuation/ package (rules + service) - Extracted from PuncAra.py: preprocessing, postprocessing, chunking logic - PunctuationChecker: lazy-loaded EncoderDecoderModel from bayan10/PuncAra-v1 - Wired into /api/analyze as Step 3 (Spelling -> Grammar -> Punctuation) - Updated /api/punctuation standalone endpoint - Updated /api/health to report punctuation status - Dockerfile pre-downloads PuncAra-v1 weights during build - Increased gunicorn timeout to 300s for full pipeline

Dockerfile

@@ -48,6 +48,17 @@ print('Spelling model + MLM cached!'); \
 # 3. Grammar — camel-tools MLE disambiguator data
 RUN camel_data -i light
 
+# 4. Punctuation model (PuncAra-v1 — EncoderDecoderModel)
+RUN python -c "\
+from transformers import EncoderDecoderModel, AutoTokenizer; \
+repo = 'bayan10/PuncAra-v1'; \
+print('Downloading PuncAra-v1 tokenizer...'); \
+AutoTokenizer.from_pretrained(repo); \
+print('Downloading PuncAra-v1 model...'); \
+EncoderDecoderModel.from_pretrained(repo); \
+print('PuncAra-v1 cached!'); \
+"
+
 # Copy application code
 COPY src/ ./src/
 COPY .env* ./
@@ -61,4 +72,5 @@ ENV PYTHONUNBUFFERED=1
 EXPOSE 7860
 
 # Start the app with gunicorn (single worker to minimize RAM)
-CMD ["gunicorn", "--chdir", "src", "app:app", "--bind", "0.0.0.0:7860", "--timeout", "120", "--workers", "1"]
+# Timeout 300s: full pipeline (spelling ~50s + grammar ~8s + punctuation ~30s + cold start)
+CMD ["gunicorn", "--chdir", "src", "app:app", "--bind", "0.0.0.0:7860", "--timeout", "300", "--workers", "1"]

PuncAra.py

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+"""Untitled18.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1ebBGzEo4wbwwvReea_n0PRHdfYescKcs
+"""
+
+import os
+import torch
+from transformers import EncoderDecoderModel, AutoTokenizer
+import re
+
+# تعريف الثوابت
+HF_REPO_ID = "bayan10/PuncAra-v1"
+
+# متغيرات عامة
+device = None
+test_model = None
+test_tokenizer = None
+
+def initialize_model(repo_id=HF_REPO_ID):
+    """
+    تهيئة وإعداد كرت الشاشة وتحميل النموذج والـ Tokenizer من Hugging Face Hub.
+    يتم استدعاء هذه الدالة مرة واحدة فقط في بداية تشغيل المشروع.
+    """
+    global device, test_model, test_tokenizer
+    print(f"Loading test model directly from Hugging Face Hub: {repo_id}")
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print(f"Loading test model to: {device}")
+
+    if device.type == "cuda" and not torch.cuda.is_available():
+        print("Warning: CUDA device requested, but torch.cuda.is_available() is False. Model will be loaded to CPU.")
+        device = torch.device("cpu")
+
+    test_model = EncoderDecoderModel.from_pretrained(repo_id).to(device)
+    test_tokenizer = AutoTokenizer.from_pretrained(repo_id)
+
+    # إعداد الـ Special tokens للـ Decoder والـ Encoder
+    test_model.config.decoder_start_token_id = test_tokenizer.cls_token_id
+    test_model.config.bos_token_id = test_tokenizer.cls_token_id
+    test_model.config.eos_token_id = test_tokenizer.sep_token_id
+    test_model.config.pad_token_id = test_tokenizer.pad_token_id
+    print("Model and Tokenizer loaded successfully!")
+
+def predict_chunk(text_chunk):
+    """توليد التوقعات لعلامات الترقيم لقطعة نصية صغيرة لا تتعدى الـ 128 Token."""
+    global device, test_model, test_tokenizer
+    if test_model is None or test_tokenizer is None:
+        raise RuntimeError("الموديل لم يتم تهيئته بعد. يرجى استدعاء initialize_model() أولاً.")
+
+    # تطبيق الـ Preprocessing لتنظيف التشكيل قبل دخول النص للموديل
+    text_chunk = arabic_preprocessing(text_chunk)
+
+    inputs = test_tokenizer(text_chunk, return_tensors="pt", padding=True, truncation=True, max_length=128).to(device)
+
+    outputs = test_model.generate(
+          inputs.input_ids,
+          attention_mask=inputs.attention_mask,
+          decoder_start_token_id=test_tokenizer.cls_token_id,
+          bos_token_id=test_tokenizer.cls_token_id,
+          eos_token_id=test_tokenizer.sep_token_id,
+          pad_token_id=test_tokenizer.pad_token_id,
+          max_length=128,
+          num_beams=3,
+          repetition_penalty=1.2,
+          length_penalty=1.0,
+          early_stopping=True,
+          do_sample=False
+      )
+    return test_tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+def arabic_preprocessing(text):
+    """حذف الحركات التشكيلية لتوحيد المدخلات وتسهيل عمل الموديل."""
+    arabic_diacritics = re.compile(r'[\u064B-\u0652]')
+    return re.sub(arabic_diacritics, '', text).strip()
+
+def arabic_postprocessing(text):
+    """
+    التنظيف والتحسين المطبعي وعلاج مشاكل دمج النصوص وعلامات الترقيم الزائدة.
+    """
+    if not text:
+        return text
+
+    # 1. حماية الأرقام والكسور والتوقيت من التحويل الخاطئ
+    text = re.sub(r'(?<=\d),(?=\d)', '٪TEMP_COMMA٪', text)
+    text = re.sub(r'(?<=\d):(?=\d)', '٪TEMP_COLON٪', text)
+
+    # 2. التوحيد والتعريب المطبعي للعلامات
+    text = text.replace(',', '،').replace(';', '؛').replace('?', '؟')
+
+    # 3. ضبط المسافات الداخلية للأقواس وعلامات الاقتباس العربي
+    text = re.sub(r'\(\s+', '(', text)
+    text = re.sub(r'\s+\)', ')', text)
+    text = re.sub(r'\[\s+', '[', text)
+    text = re.sub(r'\s+\]', ']', text)
+    text = re.sub(r'«\s+', '«', text)
+    text = re.sub(r'\s+»', '»', text)
+
+    # 4. منع تكرار العلامات الانفعالية عدا النقاط الثلاثية للحذف
+    text = re.sub(r'([،؛:!؟])\1+', r'\1', text)
+    text = re.sub(r'\.{4,}', '...', text)
+
+    # 5. معالجة التناقضات المباشرة الناتجة عن تجميع الـ Chunks
+    text = re.sub(r'[،؛:]+([.!؟])', r'\1', text)
+    text = re.sub(r'،؛|؛،', '؛', text)
+    text = re.sub(r'([!؟])\.', r'\1', text)
+
+    # 6. مسح علامات الترقيم العشوائية إذا ظهرت أول النص
+    text = re.sub(r'^[،؛:!؟. \t]+', '', text)
+
+    # 7. ضمان مسافة فارغة واحدة بعد علامة الترقيم إذا تبعها كلام
+    text = re.sub(r'([،؛:!؟.])(?=\S)', r'\1 ', text)
+
+    # 8. إعادة الأرقام والكسور والتوقيت المحمية إلى أصلها
+    text = text.replace('٪TEMP_COMMA٪', ',').replace('٪TEMP_COLON٪', ':')
+
+    # 9. إلصاق علامات الترقيم بالكلمة السابقة لها مباشرة
+    text = re.sub(r'\s+([،؛:!؟.])', r'\1', text)
+
+    # 10. إزالة المسافات المتكررة الأفقية فقط (بدون لمس السطور الجديدة)
+    text = re.sub(r'[ \t]+', ' ', text).strip()
+    return text
+
+def fix_punctuation(text):
+    """معالجة الفقرة الواحدة الطويلة عبر تقسيمها لقطع غير متداخلة لمنع التكرار."""
+    words = text.split()
+    total_words = len(words)
+
+    # جعل حجم الخطوة مساوياً لحجم النافذة يمنع تكرار الكلمات تماماً
+    window_size = 50
+    stride = 50
+
+    if total_words <= window_size:
+        result = predict_chunk(text)
+    else:
+        segments_output = []
+
+        for i in range(0, total_words, stride):
+            chunk_words = words[i : i + window_size]
+            chunk_text = " ".join(chunk_words)
+
+            if not chunk_text.strip():
+                continue
+
+            processed_segment = predict_chunk(chunk_text).strip()
+
+            # مسح علامات الترقيم الناتجة عن القص الإجباري بين القطع
+            is_last_segment = (i + window_size) >= total_words
+            if not is_last_segment:
+                punctuation_marks = ".?!،؛:؟!"
+                if processed_segment and processed_segment[-1] in punctuation_marks:
+                    # نمسح العلامة تماماً لأن السياق مستمر في القطعة اللي بعدها
+                    processed_segment = processed_segment[:-1]
+
+            segments_output.append(processed_segment)
+
+        result = " ".join(segments_output)
+
+    # تنظيف المسافات الزائدة والتكرار إن وجد
+    result = re.sub(r'\s+', ' ', result).strip()
+    return result
+
+def process_full_document(text):
+    if not text:
+        return text
+
+    # تقسيم بناءً على السطور الجديدة وتنظيف الأسطر الفارغة
+    paragraphs = [p.strip() for p in text.split('\n') if p.strip()]
+    processed_paragraphs = []
+
+    for paragraph in paragraphs:
+        # معالجة الفقرة المستقلة
+        punctuated_paragraph = fix_punctuation(paragraph)
+        cleaned_paragraph = arabic_postprocessing(punctuated_paragraph)
+        processed_paragraphs.append(cleaned_paragraph)
+
+    # الدمج بسطرين متباعدين لضمان الفصل البصري التام بين الفقرات
+    return "\n\n".join(processed_paragraphs)

src/app.py

@@ -156,7 +156,7 @@ def health_check():
                 'spelling': _spelling_available(),
                 'autocomplete': False,
                 'grammar': _grammar_available(),
-                'punctuation': False
+                'punctuation': _punctuation_available()
             },
             'note': 'Free tier: summarization local, other models return input unchanged',
             'supabase': {
@@ -241,6 +241,15 @@ def _grammar_available():
         return False
 
 
+def _punctuation_available():
+    """Check if punctuation model is loaded (without triggering lazy load)."""
+    try:
+        from nlp.punctuation.punctuation_service import is_loaded
+        return is_loaded()
+    except Exception:
+        return False
+
+
 @app.route('/api/spelling', methods=['POST'])
 def spelling_correction():
     """
@@ -506,42 +515,47 @@ def grammar_correction():
 @app.route('/api/punctuation', methods=['POST'])
 def add_punctuation():
     """
-    Add punctuation to Arabic text.
-    
-    Expected JSON payload:
+    Add punctuation to Arabic text using PuncAra-v1.
+
+    Request JSON:
     {
         "text": "Arabic text without punctuation"
     }
+
+    Response JSON:
+    {
+        "status": "success",
+        "original_text": "...",
+        "corrected_text": "..."
+    }
     """
-    if not USE_HF_API and punctuation_model is None:
-        return jsonify({
-            'error': 'Punctuation model not loaded. Please check server logs.',
-            'status': 'error'
-        }), 503
-    
     try:
         if not request.is_json:
             return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
-        
+
         data = request.get_json()
         text = data.get('text', '').strip()
-        
+
         if not text:
             return jsonify({'error': 'Text is required', 'status': 'error'}), 400
-        
+
         logger.info(f"Adding punctuation for text of length: {len(text)}")
-        if USE_HF_API:
-            punctuated = hf_add_punctuation(text)
-        else:
-            punctuated = punctuation_model.add_punctuation(text)
-        
+        from nlp.punctuation.punctuation_service import get_punctuation_model
+        punc_checker = get_punctuation_model()
+        punctuated = punc_checker.correct(text)
+
         return jsonify({
-            'punctuated': punctuated,
-            'status': 'success',
-            'original_length': len(text),
-            'punctuated_length': len(punctuated)
+            'original_text': text,
+            'corrected_text': punctuated,
+            'status': 'success'
         })
-    
+
+    except RuntimeError as e:
+        logger.error(f"Punctuation model error: {e}")
+        return jsonify({
+            'error': f'Punctuation model unavailable: {str(e)[:200]}',
+            'status': 'error'
+        }), 503
     except Exception as e:
         logger.error(f"Error during punctuation: {str(e)}")
         logger.error(traceback.format_exc())
@@ -893,31 +907,29 @@ def analyze_text():
         except Exception as e:
             logger.error(f"[ANALYZE] Grammar failed: {e}")
 
-        # 3. Punctuation (runs on grammar-corrected text)
-        has_punctuation = USE_HF_API or punctuation_model
-        if has_punctuation:
-            try:
-                t0 = time.time()
-                logger.info(f"[ANALYZE] Step 3: Punctuation starting...")
-                if USE_HF_API:
-                    corrected_punc = hf_add_punctuation(current_text)
-                else:
-                    corrected_punc = punctuation_model.add_punctuation(current_text)
-                logger.info(f"[ANALYZE] Step 3: Punctuation done in {time.time()-t0:.2f}s")
-                if corrected_punc != current_text:
-                    diffs = get_word_diffs(current_text, corrected_punc)
-                    for d in diffs:
-                        orig_start, orig_end = map_range_to_original(d['start'], d['end'])
-                        suggestions.append({
-                            'start': orig_start,
-                            'end': orig_end,
-                            'original': text[orig_start:orig_end],
-                            'correction': d['correction'],
-                            'type': 'punctuation'
-                        })
-                    current_text = corrected_punc
-            except Exception as e:
-                logger.error(f"[ANALYZE] Punctuation failed: {e}")
+        # 3. Punctuation (runs on grammar-corrected text — PuncAra-v1 local model)
+        try:
+            t0 = time.time()
+            logger.info(f"[ANALYZE] Step 3: Punctuation starting...")
+            from nlp.punctuation.punctuation_service import get_punctuation_model
+            punc_checker = get_punctuation_model()
+            corrected_punc = punc_checker.correct(current_text)
+            logger.info(f"[ANALYZE] Step 3: Punctuation done in {time.time()-t0:.2f}s")
+            if corrected_punc != current_text:
+                diffs = get_word_diffs(current_text, corrected_punc)
+                for d in diffs:
+                    orig_start, orig_end = map_range_to_original(d['start'], d['end'])
+                    suggestions.append({
+                        'start': orig_start,
+                        'end': orig_end,
+                        'original': text[orig_start:orig_end],
+                        'correction': d['correction'],
+                        'type': 'punctuation'
+                    })
+                mappers.append(OffsetMapper(current_text, corrected_punc))
+                current_text = corrected_punc
+        except Exception as e:
+            logger.error(f"[ANALYZE] Punctuation failed: {e}")
 
         total_time = time.time() - total_start
 

src/nlp/punctuation/__init__.py

@@ -0,0 +1 @@
+# PuncAra punctuation package

src/nlp/punctuation/punctuation_rules.py

@@ -0,0 +1,63 @@
+# PuncAra — Arabic Punctuation Restoration Rules
+# Extracted from PuncAra.py — preprocessing + postprocessing + chunking logic.
+# All classes are imported by punctuation_service.py.
+
+import re
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def arabic_preprocessing(text: str) -> str:
+    """Remove Arabic diacritics to normalize input for the model."""
+    arabic_diacritics = re.compile(r'[\u064B-\u0652]')
+    return re.sub(arabic_diacritics, '', text).strip()
+
+
+def arabic_postprocessing(text: str) -> str:
+    """
+    Typographic cleanup and punctuation normalization after model inference.
+    Handles: bracket spacing, duplicate marks, chunk-join artifacts, etc.
+    """
+    if not text:
+        return text
+
+    # 1. Protect numbers/fractions/time from incorrect conversion
+    text = re.sub(r'(?<=\d),(?=\d)', '٪TEMP_COMMA٪', text)
+    text = re.sub(r'(?<=\d):(?=\d)', '٪TEMP_COLON٪', text)
+
+    # 2. Arabize typographic marks
+    text = text.replace(',', '،').replace(';', '؛').replace('?', '؟')
+
+    # 3. Fix internal spacing for brackets and Arabic quotes
+    text = re.sub(r'\(\s+', '(', text)
+    text = re.sub(r'\s+\)', ')', text)
+    text = re.sub(r'\[\s+', '[', text)
+    text = re.sub(r'\s+\]', ']', text)
+    text = re.sub(r'«\s+', '«', text)
+    text = re.sub(r'\s+»', '»', text)
+
+    # 4. Remove repeated emotional marks (except ellipsis)
+    text = re.sub(r'([،؛:!؟])\1+', r'\1', text)
+    text = re.sub(r'\.{4,}', '...', text)
+
+    # 5. Fix chunk-join contradictions
+    text = re.sub(r'[،؛:]+([.!؟])', r'\1', text)
+    text = re.sub(r'،؛|؛،', '؛', text)
+    text = re.sub(r'([!؟])\.', r'\1', text)
+
+    # 6. Remove stray leading punctuation
+    text = re.sub(r'^[،؛:!؟. \t]+', '', text)
+
+    # 7. Ensure single space after punctuation before text
+    text = re.sub(r'([،؛:!؟.])(?=\S)', r'\1 ', text)
+
+    # 8. Restore protected numbers
+    text = text.replace('٪TEMP_COMMA٪', ',').replace('٪TEMP_COLON٪', ':')
+
+    # 9. Attach punctuation to preceding word
+    text = re.sub(r'\s+([،؛:!؟.])', r'\1', text)
+
+    # 10. Collapse horizontal spaces only
+    text = re.sub(r'[ \t]+', ' ', text).strip()
+    return text

src/nlp/punctuation/punctuation_service.py

@@ -0,0 +1,185 @@
+"""
+Punctuation Service — Lazy-loaded Arabic punctuation restoration.
+
+Uses:
+  1. bayan10/PuncAra-v1 (EncoderDecoderModel — local, seq2seq)
+  2. Rule-based pre/post-processing from punctuation_rules.py
+
+Model loaded on first request and kept in memory.
+"""
+
+import logging
+import time
+import torch
+import re
+
+logger = logging.getLogger(__name__)
+
+# ── Lazy-loaded singletons ──
+_punctuation_checker = None
+_load_error = None
+
+HF_REPO_ID = "bayan10/PuncAra-v1"
+
+
+class PunctuationChecker:
+    """
+    Arabic punctuation restoration pipeline:
+      1. Preprocessing (remove diacritics)
+      2. Model inference (chunked, windowed — 50 words/chunk)
+      3. Postprocessing (typographic cleanup)
+    """
+
+    def __init__(self, model, tokenizer, device):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.device = device
+
+    def _predict_chunk(self, text_chunk: str) -> str:
+        """Run model inference on a single chunk (max 128 tokens)."""
+        from nlp.punctuation.punctuation_rules import arabic_preprocessing
+
+        text_chunk = arabic_preprocessing(text_chunk)
+
+        inputs = self.tokenizer(
+            text_chunk, return_tensors="pt",
+            padding=True, truncation=True, max_length=128
+        ).to(self.device)
+
+        with torch.no_grad():
+            outputs = self.model.generate(
+                inputs.input_ids,
+                attention_mask=inputs.attention_mask,
+                decoder_start_token_id=self.tokenizer.cls_token_id,
+                bos_token_id=self.tokenizer.cls_token_id,
+                eos_token_id=self.tokenizer.sep_token_id,
+                pad_token_id=self.tokenizer.pad_token_id,
+                max_length=128,
+                num_beams=3,
+                repetition_penalty=1.2,
+                length_penalty=1.0,
+                early_stopping=True,
+                do_sample=False
+            )
+
+        return self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+    def _fix_punctuation(self, text: str) -> str:
+        """Process a paragraph using non-overlapping window chunking."""
+        words = text.split()
+        total_words = len(words)
+        window_size = 50
+        stride = 50
+
+        if total_words <= window_size:
+            return self._predict_chunk(text)
+
+        segments_output = []
+        for i in range(0, total_words, stride):
+            chunk_words = words[i: i + window_size]
+            chunk_text = " ".join(chunk_words)
+            if not chunk_text.strip():
+                continue
+
+            processed_segment = self._predict_chunk(chunk_text).strip()
+
+            # Remove trailing punctuation from non-last segments (context continues)
+            is_last_segment = (i + window_size) >= total_words
+            if not is_last_segment:
+                punctuation_marks = ".?!،؛:؟!"
+                if processed_segment and processed_segment[-1] in punctuation_marks:
+                    processed_segment = processed_segment[:-1]
+
+            segments_output.append(processed_segment)
+
+        result = " ".join(segments_output)
+        result = re.sub(r'\s+', ' ', result).strip()
+        return result
+
+    def correct(self, text: str) -> str:
+        """
+        Run full punctuation restoration on text.
+        Handles multi-paragraph documents.
+        Returns punctuated text, or original text on failure.
+        """
+        if not text or not text.strip():
+            return text
+
+        try:
+            from nlp.punctuation.punctuation_rules import arabic_postprocessing
+
+            # Split into paragraphs
+            paragraphs = [p.strip() for p in text.split('\n') if p.strip()]
+            processed_paragraphs = []
+
+            for paragraph in paragraphs:
+                punctuated = self._fix_punctuation(paragraph)
+                cleaned = arabic_postprocessing(punctuated)
+                processed_paragraphs.append(cleaned)
+
+            result = "\n\n".join(processed_paragraphs)
+            logger.info(f"Punctuation output: '{result[:80]}...' (input: '{text[:80]}...')")
+            return result
+
+        except Exception as e:
+            logger.error(f"Punctuation correction failed: {e}")
+            return text
+
+
+def get_punctuation_model():
+    """
+    Lazy-load the punctuation model on first call.
+    Returns the PunctuationChecker instance, or raises RuntimeError if loading fails.
+    """
+    global _punctuation_checker, _load_error
+
+    if _punctuation_checker is not None:
+        return _punctuation_checker
+
+    if _load_error is not None:
+        raise RuntimeError(f"Punctuation model previously failed to load: {_load_error}")
+
+    try:
+        t0 = time.time()
+        logger.info("Loading PuncAra-v1 punctuation model (lazy init)...")
+
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        logger.info(f"Punctuation model device: {device}")
+
+        from transformers import EncoderDecoderModel, AutoTokenizer
+
+        logger.info(f"Loading model from HF Hub: {HF_REPO_ID}")
+        model = EncoderDecoderModel.from_pretrained(HF_REPO_ID)
+        tokenizer = AutoTokenizer.from_pretrained(HF_REPO_ID)
+
+        # Configure special tokens
+        model.config.decoder_start_token_id = tokenizer.cls_token_id
+        model.config.bos_token_id = tokenizer.cls_token_id
+        model.config.eos_token_id = tokenizer.sep_token_id
+        model.config.pad_token_id = tokenizer.pad_token_id
+
+        model = model.to(device)
+        model.eval()
+
+        _punctuation_checker = PunctuationChecker(model, tokenizer, device)
+
+        elapsed = time.time() - t0
+        logger.info(f"PuncAra-v1 ready in {elapsed:.1f}s")
+        return _punctuation_checker
+
+    except Exception as e:
+        import traceback
+        _load_error = str(e)
+        logger.error(f"Failed to load punctuation model: {e}")
+        logger.error(traceback.format_exc())
+        raise RuntimeError(f"Punctuation model load failed: {e}")
+
+
+def is_loaded() -> bool:
+    """Check if the punctuation model is loaded."""
+    return _punctuation_checker is not None
+
+
+def get_load_error() -> str:
+    """Return the last load error, or empty string."""
+    return _load_error or ""