Update app.py

app.py

@@ -1,53 +1,78 @@
 from fastapi import FastAPI, File, UploadFile, Form
-from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from langdetect import detect
+import re
 
 app = FastAPI()
 
-model_name = "facebook/m2m100_418M"
-model = M2M100ForConditionalGeneration.from_pretrained(model_name)
-tokenizer = M2M100Tokenizer.from_pretrained(model_name)
+MODEL_NAME = "facebook/nllb-200-600M"
+model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+
+def split_text_lines(text, max_chunk_length=900):
+    # تقسيم ذكي مع الحفاظ على أسطر strings
+    lines = text.splitlines()
+    chunks = []
+    chunk = ""
+    for line in lines:
+        if len(chunk) + len(line) < max_chunk_length:
+            chunk += line + "\n"
+        else:
+            if chunk.strip():
+                chunks.append(chunk.strip())
+            chunk = line + "\n"
+    if chunk.strip():
+        chunks.append(chunk.strip())
+    return chunks
+
+def batch_translate(texts, src_lang, tgt_lang):
+    # ترجمة سريعة batch
+    results = []
+    batch_size = 8  # يمكنك زيادة العدد حسب موارد السيرفر
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i+batch_size]
+        tokenizer.src_lang = src_lang
+        inputs = tokenizer(batch, return_tensors="pt", padding=True, truncation=True, max_length=1024)
+        generated = model.generate(**inputs, forced_bos_token_id=tokenizer.convert_lang_to_id(tgt_lang))
+        translated = tokenizer.batch_decode(generated, skip_special_tokens=True)
+        results.extend(translated)
+    return results
+
+def detect_language(text):
+    # كشف لغة ذكي (يعمل على أول chunk)
+    sample = text[:2000] if len(text) > 2000 else text
+    lang = detect(sample)
+    # وفق أكواد NLLB المتوافقة (تعديل سريع)
+    lang_map = {"en": "eng_Latn", "ar": "arb_Arab", "fr": "fra_Latn", "hi": "hin_Deva", "es": "spa_Latn", "de": "deu_Latn"}
+    return lang_map.get(lang, "eng_Latn")
 
 @app.post("/translate-text")
 async def translate_text(
     text: str = Form(...),
-    source_lang: str = Form(...),
     target_lang: str = Form(...)
 ):
-    tokenizer.src_lang = source_lang
-    encoded = tokenizer(text, return_tensors="pt")
-    generated = model.generate(**encoded, forced_bos_token_id=tokenizer.get_lang_id(target_lang))
-    translated = tokenizer.batch_decode(generated, skip_special_tokens=True)[0]
-    return {"translated_text": translated}
+    source_lang = detect_language(text)
+    texts = re.split(r'(?<=[.!?\n])\s+', text.strip())
+    chunks = []
+    cur_chunk = ""
+    for sentence in texts:
+        if len(cur_chunk) + len(sentence) < 900:
+            cur_chunk += sentence + " "
+        else:
+            chunks.append(cur_chunk.strip())
+            cur_chunk = sentence + " "
+    if cur_chunk.strip(): chunks.append(cur_chunk.strip())
+    translated = batch_translate(chunks, source_lang, target_lang)
+    return {"translated_text": "\n".join(translated)}
 
 @app.post("/translate-file")
 async def translate_file(
     file: UploadFile = File(...),
-    source_lang: str = Form(...),
     target_lang: str = Form(...)
 ):
     contents = await file.read()
     original_text = contents.decode()
-
-    # قسم النص الأصلي إلى شرائح في حدود 900 حرف تقريباً لكل شريحة (يمكنك ضبط الرقم حسب تجربة الأداء)
-    lines = original_text.splitlines()
-    chunks = []
-    chunk = ""
-    max_chunk_length = 900
-    for line in lines:
-        if len(chunk) + len(line) < max_chunk_length:
-            chunk += line + "\n"
-        else:
-            chunks.append(chunk.strip())
-            chunk = line + "\n"
-    if chunk:
-        chunks.append(chunk.strip())
-
-    result = ""
-    for chunk in chunks:
-        tokenizer.src_lang = source_lang
-        encoded = tokenizer(chunk, return_tensors="pt")
-        generated = model.generate(**encoded, forced_bos_token_id=tokenizer.get_lang_id(target_lang))
-        translated = tokenizer.batch_decode(generated, skip_special_tokens=True)[0]
-        result += translated + "\n"
-
-    return {"translated_text": result}
+    source_lang = detect_language(original_text)
+    lines = split_text_lines(original_text)
+    translated_lines = batch_translate(lines, source_lang, target_lang)
+    return {"translated_text": "\n".join(translated_lines)}