Update main.py
Browse files
main.py
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
|
|
| 1 |
import torch
|
| 2 |
import torch.nn as nn
|
| 3 |
-
import math
|
| 4 |
-
import re
|
| 5 |
from fastapi import FastAPI
|
| 6 |
from pydantic import BaseModel
|
|
|
|
| 7 |
|
| 8 |
# ==========================================
|
| 9 |
-
# 1. الثوابت (Constants)
|
| 10 |
# ==========================================
|
|
|
|
|
|
|
| 11 |
FATHA = '\u064E'
|
| 12 |
DAMMA = '\u064F'
|
| 13 |
KASRA = '\u0650'
|
|
@@ -26,13 +28,12 @@ DIACRITIC_CLASSES = [
|
|
| 26 |
]
|
| 27 |
IDX_TO_DIAC = {i: d for i, d in enumerate(DIACRITIC_CLASSES)}
|
| 28 |
NUM_CLASSES = 15
|
| 29 |
-
DEVICE = torch.device('cpu') # إجبار العمل على المعالج العادي للسيرفر المجاني
|
| 30 |
|
| 31 |
def remove_diacritics(text):
|
| 32 |
return ''.join(c for c in text if c not in ALL_DIACRITICS)
|
| 33 |
|
| 34 |
# ==========================================
|
| 35 |
-
# 2. بنية
|
| 36 |
# ==========================================
|
| 37 |
class MultiHeadSelfAttention(nn.Module):
|
| 38 |
def __init__(self, hidden_dim, num_heads=8, dropout=0.1):
|
|
@@ -116,18 +117,16 @@ class BiLSTMAttentionDiacritizer(nn.Module):
|
|
| 116 |
return self.classifier(lstm_out)
|
| 117 |
|
| 118 |
# ==========================================
|
| 119 |
-
# 3. تحميل النم
|
| 120 |
# ==========================================
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
char_to_idx = checkpoint['char_to_idx']
|
| 126 |
-
VOCAB_SIZE = len(char_to_idx)
|
| 127 |
|
| 128 |
-
# تهيئة النموذج بنفس الإعدادات التي تدرب عليها
|
| 129 |
tashkeel_model = BiLSTMAttentionDiacritizer(
|
| 130 |
-
vocab_size=
|
| 131 |
embed_dim=256,
|
| 132 |
hidden_dim=256,
|
| 133 |
num_lstm_layers=3,
|
|
@@ -137,12 +136,20 @@ tashkeel_model = BiLSTMAttentionDiacritizer(
|
|
| 137 |
dropout=0.3
|
| 138 |
).to(DEVICE)
|
| 139 |
|
| 140 |
-
# تركيب الأوزان
|
| 141 |
tashkeel_model.load_state_dict(checkpoint['model_state_dict'])
|
| 142 |
-
tashkeel_model.eval()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
|
| 144 |
# ==========================================
|
| 145 |
-
# 4. د
|
| 146 |
# ==========================================
|
| 147 |
def diacritize_text(text, model, char_to_idx, max_len=200):
|
| 148 |
clean = remove_diacritics(text)
|
|
@@ -180,16 +187,33 @@ def diacritize_text(text, model, char_to_idx, max_len=200):
|
|
| 180 |
if d: result.append(d)
|
| 181 |
return ''.join(result)
|
| 182 |
|
|
|
|
|
|
|
|
|
|
| 183 |
app = FastAPI()
|
| 184 |
|
| 185 |
class TextRequest(BaseModel):
|
| 186 |
text: str
|
| 187 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 188 |
@app.post("/tashkeel")
|
| 189 |
def process_tashkeel(request: TextRequest):
|
| 190 |
result = diacritize_text(request.text, tashkeel_model, char_to_idx)
|
| 191 |
-
return {"
|
| 192 |
|
| 193 |
-
@app.
|
| 194 |
-
def
|
| 195 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
import torch
|
| 3 |
import torch.nn as nn
|
|
|
|
|
|
|
| 4 |
from fastapi import FastAPI
|
| 5 |
from pydantic import BaseModel
|
| 6 |
+
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
|
| 7 |
|
| 8 |
# ==========================================
|
| 9 |
+
# 1. الإعدادات والثوابت (Constants)
|
| 10 |
# ==========================================
|
| 11 |
+
DEVICE = torch.device('cpu') # إجبار العمل على المعالج العادي للسيرفر المجاني
|
| 12 |
+
|
| 13 |
FATHA = '\u064E'
|
| 14 |
DAMMA = '\u064F'
|
| 15 |
KASRA = '\u0650'
|
|
|
|
| 28 |
]
|
| 29 |
IDX_TO_DIAC = {i: d for i, d in enumerate(DIACRITIC_CLASSES)}
|
| 30 |
NUM_CLASSES = 15
|
|
|
|
| 31 |
|
| 32 |
def remove_diacritics(text):
|
| 33 |
return ''.join(c for c in text if c not in ALL_DIACRITICS)
|
| 34 |
|
| 35 |
# ==========================================
|
| 36 |
+
# 2. بنية نموذج التشكيل (Architecture)
|
| 37 |
# ==========================================
|
| 38 |
class MultiHeadSelfAttention(nn.Module):
|
| 39 |
def __init__(self, hidden_dim, num_heads=8, dropout=0.1):
|
|
|
|
| 117 |
return self.classifier(lstm_out)
|
| 118 |
|
| 119 |
# ==========================================
|
| 120 |
+
# 3. تحميل النماذج (Model Loading)
|
| 121 |
# ==========================================
|
| 122 |
+
print("جاري تحميل نموذج التشكيل...")
|
| 123 |
+
# تأكد من أن اسم الملف هنا يطابق الملف المرفوع على Hugging Face
|
| 124 |
+
CHECKPOINT_PATH = 'Tashkeel_model.pt'
|
| 125 |
+
checkpoint = torch.load(CHECKPOINT_PATH, map_location=DEVICE)
|
| 126 |
char_to_idx = checkpoint['char_to_idx']
|
|
|
|
| 127 |
|
|
|
|
| 128 |
tashkeel_model = BiLSTMAttentionDiacritizer(
|
| 129 |
+
vocab_size=len(char_to_idx),
|
| 130 |
embed_dim=256,
|
| 131 |
hidden_dim=256,
|
| 132 |
num_lstm_layers=3,
|
|
|
|
| 136 |
dropout=0.3
|
| 137 |
).to(DEVICE)
|
| 138 |
|
|
|
|
| 139 |
tashkeel_model.load_state_dict(checkpoint['model_state_dict'])
|
| 140 |
+
tashkeel_model.eval()
|
| 141 |
+
|
| 142 |
+
print("جاري تحميل نموذج AraBART...")
|
| 143 |
+
# في حال كان مسار النموذج على Hugging Face مختلفاً، يرجى تعديل السطر التالي
|
| 144 |
+
ARABART_MODEL_NAME = "UBC-NLP/AraBART-Morph-GEC"
|
| 145 |
+
arabart_tokenizer = AutoTokenizer.from_pretrained(ARABART_MODEL_NAME)
|
| 146 |
+
arabart_model = AutoModelForSeq2SeqLM.from_pretrained(ARABART_MODEL_NAME)
|
| 147 |
+
gec_pipeline = pipeline("text2text-generation", model=arabart_model, tokenizer=arabart_tokenizer)
|
| 148 |
+
|
| 149 |
+
print("✅ تمت تهيئة جميع النماذج بنجاح!")
|
| 150 |
|
| 151 |
# ==========================================
|
| 152 |
+
# 4. دالة التشكيل المساعدة
|
| 153 |
# ==========================================
|
| 154 |
def diacritize_text(text, model, char_to_idx, max_len=200):
|
| 155 |
clean = remove_diacritics(text)
|
|
|
|
| 187 |
if d: result.append(d)
|
| 188 |
return ''.join(result)
|
| 189 |
|
| 190 |
+
# ==========================================
|
| 191 |
+
# 5. إعداد الخادم ومسارات الـ API
|
| 192 |
+
# ==========================================
|
| 193 |
app = FastAPI()
|
| 194 |
|
| 195 |
class TextRequest(BaseModel):
|
| 196 |
text: str
|
| 197 |
|
| 198 |
+
@app.get("/")
|
| 199 |
+
def read_root():
|
| 200 |
+
return {
|
| 201 |
+
"status": "online",
|
| 202 |
+
"message": "مرحباً بك في واجهة برمجة تطبيقات نظام نحو (NAHW) للمعالجة الذكية للنصوص",
|
| 203 |
+
"endpoints": ["/tashkeel", "/spell-check", "/grammar-check"]
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
@app.post("/tashkeel")
|
| 207 |
def process_tashkeel(request: TextRequest):
|
| 208 |
result = diacritize_text(request.text, tashkeel_model, char_to_idx)
|
| 209 |
+
return {"result_text": result}
|
| 210 |
|
| 211 |
+
@app.post("/spell-check")
|
| 212 |
+
def spell_check(request: TextRequest):
|
| 213 |
+
result = gec_pipeline(request.text)
|
| 214 |
+
return {"result_text": result[0]['generated_text']}
|
| 215 |
+
|
| 216 |
+
@app.post("/grammar-check")
|
| 217 |
+
def grammar_check(request: TextRequest):
|
| 218 |
+
result = gec_pipeline(request.text)
|
| 219 |
+
return {"result_text": result[0]['generated_text']}
|