EasyOCR + Traductor

app.py

@@ -1,6 +1,6 @@
 """
-app.py  —  FastAPI + EasyOCR + Gemini zerbitzaria
-Postzuzenketa testuingurua ulertzen duen LLM bidez.
+app.py  —  FastAPI + EasyOCR + Gemini + NLLB + HiTZ zerbitzaria
+OCR + postzuzenketa (Gemini 2.5 Flash) + itzulpena (NLLB-200 + HiTZ).
 """
 
 import io
@@ -11,16 +11,21 @@ from contextlib import asynccontextmanager
 import easyocr
 import httpx
 import numpy as np
+import torch
 from deskew import determine_skew
 from fastapi import FastAPI, File, Form, HTTPException, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from PIL import Image
 from skimage.transform import rotate
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+# ────────────────────────────────────────────────────────────────────────────
+# OCR konfigurazioa
+# ────────────────────────────────────────────────────────────────────────────
 SCRIPTS = {
     "latin":      ["en","es","fr","de","it","pt","nl","pl","cs","sk","hr",
                    "ro","hu","lt","lv","et","sv","da","no","is","mt","sq","tr","vi"],
@@ -35,7 +40,9 @@ SCRIPTS = {
 
 MAX_SIDE = 1500
 
+# ────────────────────────────────────────────────────────────────────────────
 # Gemini konfigurazioa
+# ────────────────────────────────────────────────────────────────────────────
 GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "")
 GEMINI_MODEL = "gemini-2.5-flash"
 GEMINI_URL = (
@@ -105,7 +112,230 @@ Corrected text:"""
 
 readers: dict = {}
 
+# ────────────────────────────────────────────────────────────────────────────
+# Itzulpen-eredu konfigurazioa
+# ────────────────────────────────────────────────────────────────────────────
+NLLB_MODEL_NAME = "facebook/nllb-200-distilled-600M"
+
+HITZ_PAIRS = {
+    ("en", "eu"): "HiTZ/mt-hitz-en-eu",
+    ("eu", "en"): "HiTZ/mt-hitz-eu-en",
+    ("es", "eu"): "HiTZ/mt-hitz-es-eu",
+    ("eu", "es"): "HiTZ/mt-hitz-eu-es",
+    ("gl", "eu"): "HiTZ/mt-hitz-gl-eu",
+    ("ca", "eu"): "HiTZ/mt-hitz-ca-eu",
+}
+
+# ISO 639-1 (edo 639-3 alternatibarik ez badago) → NLLB-200 kode osoa
+ISO_TO_NLLB = {
+    "ace": "ace_Latn",
+    "acm": "acm_Arab",
+    "acq": "acq_Arab",
+    "aeb": "aeb_Arab",
+    "af":  "afr_Latn",
+    "ajp": "ajp_Arab",
+    "ak":  "aka_Latn",
+    "am":  "amh_Ethi",
+    "apc": "apc_Arab",
+    "arb": "arb_Arab",
+    "ars": "ars_Arab",
+    "ary": "ary_Arab",
+    "arz": "arz_Arab",
+    "as":  "asm_Beng",
+    "ast": "ast_Latn",
+    "awa": "awa_Deva",
+    "ay":  "ayr_Latn",
+    "azb": "azb_Arab",
+    "az":  "azj_Latn",
+    "ba":  "bak_Cyrl",
+    "bm":  "bam_Latn",
+    "ban": "ban_Latn",
+    "be":  "bel_Cyrl",
+    "bem": "bem_Latn",
+    "bn":  "ben_Beng",
+    "bho": "bho_Deva",
+    "bjn": "bjn_Latn",
+    "bo":  "bod_Tibt",
+    "bs":  "bos_Latn",
+    "bug": "bug_Latn",
+    "bg":  "bul_Cyrl",
+    "ca":  "cat_Latn",
+    "ceb": "ceb_Latn",
+    "cs":  "ces_Latn",
+    "cjk": "cjk_Latn",
+    "ckb": "ckb_Arab",
+    "crh": "crh_Latn",
+    "cy":  "cym_Latn",
+    "da":  "dan_Latn",
+    "de":  "deu_Latn",
+    "dik": "dik_Latn",
+    "dyu": "dyu_Latn",
+    "dz":  "dzo_Tibt",
+    "el":  "ell_Grek",
+    "en":  "eng_Latn",
+    "eo":  "epo_Latn",
+    "et":  "est_Latn",
+    "eu":  "eus_Latn",
+    "ee":  "ewe_Latn",
+    "fo":  "fao_Latn",
+    "fj":  "fij_Latn",
+    "fi":  "fin_Latn",
+    "fon": "fon_Latn",
+    "fr":  "fra_Latn",
+    "fur": "fur_Latn",
+    "fuv": "fuv_Latn",
+    "gd":  "gla_Latn",
+    "ga":  "gle_Latn",
+    "gl":  "glg_Latn",
+    "gn":  "grn_Latn",
+    "gu":  "guj_Gujr",
+    "ht":  "hat_Latn",
+    "ha":  "hau_Latn",
+    "he":  "heb_Hebr",
+    "hi":  "hin_Deva",
+    "hne": "hne_Deva",
+    "hr":  "hrv_Latn",
+    "hu":  "hun_Latn",
+    "hy":  "hye_Armn",
+    "ig":  "ibo_Latn",
+    "ilo": "ilo_Latn",
+    "id":  "ind_Latn",
+    "is":  "isl_Latn",
+    "it":  "ita_Latn",
+    "jv":  "jav_Latn",
+    "ja":  "jpn_Jpan",
+    "kab": "kab_Latn",
+    "kac": "kac_Latn",
+    "kam": "kam_Latn",
+    "kn":  "kan_Knda",
+    "ks":  "kas_Arab",
+    "ka":  "kat_Geor",
+    "knc": "knc_Latn",
+    "kk":  "kaz_Cyrl",
+    "kbp": "kbp_Latn",
+    "kea": "kea_Latn",
+    "km":  "khm_Khmr",
+    "ki":  "kik_Latn",
+    "rw":  "kin_Latn",
+    "ky":  "kir_Cyrl",
+    "kmb": "kmb_Latn",
+    "kmr": "kmr_Latn",
+    "kg":  "kon_Latn",
+    "ko":  "kor_Hang",
+    "lo":  "lao_Laoo",
+    "lij": "lij_Latn",
+    "li":  "lim_Latn",
+    "ln":  "lin_Latn",
+    "lt":  "lit_Latn",
+    "lmo": "lmo_Latn",
+    "ltg": "ltg_Latn",
+    "lb":  "ltz_Latn",
+    "lua": "lua_Latn",
+    "lg":  "lug_Latn",
+    "luo": "luo_Latn",
+    "lus": "lus_Latn",
+    "lv":  "lvs_Latn",
+    "mag": "mag_Deva",
+    "mai": "mai_Deva",
+    "ml":  "mal_Mlym",
+    "mr":  "mar_Deva",
+    "min": "min_Latn",
+    "mk":  "mkd_Cyrl",
+    "mg":  "plt_Latn",
+    "mt":  "mlt_Latn",
+    "mni": "mni_Beng",
+    "mn":  "khk_Cyrl",
+    "mos": "mos_Latn",
+    "mi":  "mri_Latn",
+    "my":  "mya_Mymr",
+    "nl":  "nld_Latn",
+    "nn":  "nno_Latn",
+    "nb":  "nob_Latn",
+    "ne":  "npi_Deva",
+    "nso": "nso_Latn",
+    "nus": "nus_Latn",
+    "ny":  "nya_Latn",
+    "oc":  "oci_Latn",
+    "om":  "gaz_Latn",
+    "or":  "ory_Orya",
+    "pag": "pag_Latn",
+    "pa":  "pan_Guru",
+    "pap": "pap_Latn",
+    "fa":  "pes_Arab",
+    "pl":  "pol_Latn",
+    "pt":  "por_Latn",
+    "prs": "prs_Arab",
+    "ps":  "pbt_Arab",
+    "qu":  "quy_Latn",
+    "ro":  "ron_Latn",
+    "rn":  "run_Latn",
+    "ru":  "rus_Cyrl",
+    "sg":  "sag_Latn",
+    "sa":  "san_Deva",
+    "sat": "sat_Olck",
+    "scn": "scn_Latn",
+    "shn": "shn_Mymr",
+    "si":  "sin_Sinh",
+    "sk":  "slk_Latn",
+    "sl":  "slv_Latn",
+    "sm":  "smo_Latn",
+    "sn":  "sna_Latn",
+    "sd":  "snd_Arab",
+    "so":  "som_Latn",
+    "st":  "sot_Latn",
+    "es":  "spa_Latn",
+    "sq":  "als_Latn",
+    "sc":  "srd_Latn",
+    "sr":  "srp_Cyrl",
+    "ss":  "ssw_Latn",
+    "su":  "sun_Latn",
+    "sv":  "swe_Latn",
+    "sw":  "swh_Latn",
+    "szl": "szl_Latn",
+    "ta":  "tam_Taml",
+    "tt":  "tat_Cyrl",
+    "te":  "tel_Telu",
+    "tg":  "tgk_Cyrl",
+    "tl":  "tgl_Latn",
+    "th":  "tha_Thai",
+    "ti":  "tir_Ethi",
+    "taq": "taq_Latn",
+    "tpi": "tpi_Latn",
+    "tn":  "tsn_Latn",
+    "ts":  "tso_Latn",
+    "tk":  "tuk_Latn",
+    "tum": "tum_Latn",
+    "tr":  "tur_Latn",
+    "tw":  "twi_Latn",
+    "tzm": "tzm_Tfng",
+    "ug":  "uig_Arab",
+    "uk":  "ukr_Cyrl",
+    "umb": "umb_Latn",
+    "ur":  "urd_Arab",
+    "uz":  "uzn_Latn",
+    "vec": "vec_Latn",
+    "vi":  "vie_Latn",
+    "war": "war_Latn",
+    "wo":  "wol_Latn",
+    "xh":  "xho_Latn",
+    "yi":  "ydd_Hebr",
+    "yo":  "yor_Latn",
+    "yue": "yue_Hant",
+    "zh":  "zho_Hans",
+    "zht": "zho_Hant",
+    "ms":  "zsm_Latn",
+    "zu":  "zul_Latn",
+}
+
+# Itzulpen-eredu globalak (lifespan-en kargatzen dira)
+nllb_model = None
+nllb_tokenizer = None
+hitz_models: dict = {}  # (src, tgt) -> {"tokenizer": ..., "model": ...}
 
+
+# ────────────────────────────────────────────────────────────────────────────
+# OCR laguntzaileak
+# ────────────────────────────────────────────────────────────────────────────
 def _resize(img: Image.Image) -> Image.Image:
     w, h = img.size
     longest = max(w, h)
@@ -220,21 +450,149 @@ async def _gemini_correct(text: str) -> str:
         return text
 
 
+# ────────────────────────────────────────────────────────────────────────────
+# Itzulpen laguntzaileak
+# ────────────────────────────────────────────────────────────────────────────
+def _nllb_translate(text: str, src_nllb: str, tgt_nllb: str) -> str:
+    """NLLB-200 ereduarekin itzuli."""
+    if not text.strip():
+        return text
+    nllb_tokenizer.src_lang = src_nllb
+    inputs = nllb_tokenizer(
+        text, return_tensors="pt", truncation=True, max_length=512
+    )
+    forced_bos_token_id = nllb_tokenizer.convert_tokens_to_ids(tgt_nllb)
+    with torch.no_grad():
+        outputs = nllb_model.generate(
+            **inputs,
+            forced_bos_token_id=forced_bos_token_id,
+            max_length=512,
+            num_beams=4,
+        )
+    return nllb_tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
+
+
+def _hitz_translate(text: str, src: str, tgt: str) -> str:
+    """HiTZ eredu espezifikoarekin itzuli (euskararen norabideak)."""
+    if not text.strip():
+        return text
+    bundle = hitz_models.get((src, tgt))
+    if bundle is None:
+        raise ValueError(f"HiTZ bikote ezezaguna: {src}→{tgt}")
+    tokenizer = bundle["tokenizer"]
+    model = bundle["model"]
+    inputs = tokenizer(
+        text, return_tensors="pt", truncation=True, max_length=512
+    )
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs, max_length=512, num_beams=4
+        )
+    return tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
+
+
+def translate(text: str, src: str, tgt: str) -> str:
+    """
+    Itzulpen-bideratzaile nagusia:
+      - Kasua 1: euskara tartean ez → NLLB zuzenean.
+      - Kasua 2: euskara ↔ {es, en} → HiTZ zuzenean.
+      - Kasua 3: {gl, ca} → eu → HiTZ zuzenean.
+      - Kasua 4: euskara + beste hizkuntza bat → pibotea (en).
+      - Kasua 5: src == tgt → testua bere horretan.
+    """
+    # Kasua 5
+    if src == tgt:
+        logger.info("[TRANSLATE] src==tgt (%s) → aldaketarik gabe", src)
+        return text
+
+    # Onarpen-egiaztaketa
+    if src not in ISO_TO_NLLB:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Hizkuntza ez da onartzen: {src}",
+        )
+    if tgt not in ISO_TO_NLLB:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Hizkuntza ez da onartzen: {tgt}",
+        )
+
+    eu_involved = (src == "eu" or tgt == "eu")
+
+    # Kasua 1: euskararik ez → NLLB zuzenean
+    if not eu_involved:
+        logger.info("[TRANSLATE] NLLB zuzenean: %s → %s", src, tgt)
+        return _nllb_translate(text, ISO_TO_NLLB[src], ISO_TO_NLLB[tgt])
+
+    other = tgt if src == "eu" else src
+
+    # Kasua 2: eu ↔ {es, en}
+    if other in {"es", "en"}:
+        logger.info("[TRANSLATE] HiTZ zuzenean: %s → %s", src, tgt)
+        return _hitz_translate(text, src, tgt)
+
+    # Kasua 3: {gl, ca} → eu (norabide bakarra HiTZ ereduetan)
+    if src in {"gl", "ca"} and tgt == "eu":
+        logger.info("[TRANSLATE] HiTZ zuzenean: %s → %s", src, tgt)
+        return _hitz_translate(text, src, tgt)
+
+    # Kasua 4: pibotea ingelesetik
+    if src == "eu":
+        logger.info("[TRANSLATE] Pibotea: %s → en → %s", src, tgt)
+        intermediate = _hitz_translate(text, "eu", "en")
+        return _nllb_translate(
+            intermediate, ISO_TO_NLLB["en"], ISO_TO_NLLB[tgt]
+        )
+    else:
+        # tgt == "eu"
+        logger.info("[TRANSLATE] Pibotea: %s → en → %s", src, tgt)
+        intermediate = _nllb_translate(
+            text, ISO_TO_NLLB[src], ISO_TO_NLLB["en"]
+        )
+        return _hitz_translate(intermediate, "en", "eu")
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# Lifespan: ereduen karga
+# ────────────────────────────────────────────────────────────────────────────
 @asynccontextmanager
 async def lifespan(app: FastAPI):
+    # EasyOCR irakurleak
     for name, langs in SCRIPTS.items():
         logger.info("Reader kargatzen: %s %s", name, langs)
         readers[name] = easyocr.Reader(langs, gpu=False)
+
+    # NLLB-200
+    global nllb_model, nllb_tokenizer
+    logger.info("[TRANSLATE] NLLB eredua kargatzen: %s", NLLB_MODEL_NAME)
+    nllb_tokenizer = AutoTokenizer.from_pretrained(NLLB_MODEL_NAME)
+    nllb_model = AutoModelForSeq2SeqLM.from_pretrained(NLLB_MODEL_NAME)
+    nllb_model.eval()
+    logger.info("[TRANSLATE] NLLB prest (CPU).")
+
+    # HiTZ
+    for (src, tgt), repo in HITZ_PAIRS.items():
+        logger.info(
+            "[TRANSLATE] HiTZ eredua kargatzen: %s (%s→%s)", repo, src, tgt
+        )
+        tok = AutoTokenizer.from_pretrained(repo)
+        mod = AutoModelForSeq2SeqLM.from_pretrained(repo)
+        mod.eval()
+        hitz_models[(src, tgt)] = {"tokenizer": tok, "model": mod}
+    logger.info("[TRANSLATE] HiTZ eredu guztiak prest (%d).", len(hitz_models))
+
     if GEMINI_API_KEY:
         logger.info("Gemini konfiguratuta: %s", GEMINI_MODEL)
     else:
         logger.warning("Gemini API key gabe — zuzenketak ez dira aplikatuko")
+
     logger.info("Sistema prest.")
     yield
     readers.clear()
+    hitz_models.clear()
 
 
-app = FastAPI(title="OCR API", version="13.0.0", lifespan=lifespan)
+app = FastAPI(title="OCR + Itzulpena API", version="14.0.0", lifespan=lifespan)
 
 app.add_middleware(
     CORSMiddleware,
@@ -244,12 +602,17 @@ app.add_middleware(
 )
 
 
+# ────────────────────────────────────────────────────────────────────────────
+# Endpoints
+# ────────────────────────────────────────────────────────────────────────────
 @app.get("/")
 async def health_check():
     return {
         "status": "ok",
         "scripts": list(readers.keys()),
         "gemini": bool(GEMINI_API_KEY),
+        "nllb": nllb_model is not None,
+        "hitz_pairs": [f"{s}-{t}" for (s, t) in hitz_models.keys()],
     }
 
 
@@ -285,4 +648,36 @@ async def predict(
         text = raw_text
 
     logger.info("[RESPONSE] Karaktereak: %d", len(text))
-    return JSONResponse(content={"text": text})
+    return JSONResponse(content={"text": text})
+
+
+@app.post("/translate")
+def translate_endpoint(
+    text: str = Form(...),
+    source_lang: str = Form(...),
+    target_lang: str = Form(...),
+):
+    """
+    Itzulpen-eskaria. Form-eko parametroak:
+      - text: itzuli beharreko testua
+      - source_lang: jatorrizko hizkuntza (ISO 639-1, edo 639-3 alternatibarik ez badago)
+      - target_lang: helburuko hizkuntza
+    Erantzuna: { "translation": "..." }
+    """
+    logger.info(
+        "[TRANSLATE] Eskaria jaso: %s → %s (%d kar.)",
+        source_lang, target_lang, len(text),
+    )
+
+    try:
+        translation = translate(text, source_lang, target_lang)
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error("[TRANSLATE] Errorea: %s", e)
+        raise HTTPException(
+            status_code=500, detail=f"Itzulpen-errorea: {e}"
+        )
+
+    logger.info("[TRANSLATE] Egina (%d kar.)", len(translation))
+    return JSONResponse(content={"translation": translation})

requirements.txt

@@ -6,4 +6,8 @@ easyocr==1.7.1
 numpy==1.26.4
 deskew==1.3.2
 scikit-image==0.22.0
-httpx==0.27.0
+httpx==0.27.0
+transformers==4.44.2
+torch==2.2.2
+sentencepiece==0.2.0
+sacremoses==0.1.1