AMWAL v1: spaCy Arabic Financial NER with normalization pipeline

.gitattributes

@@ -1,35 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+spacy_model/** filter=lfs diff=lfs merge=lfs -text

amwal.py

@@ -0,0 +1,70 @@
+import re
+import os
+import spacy
+from huggingface_hub import snapshot_download
+
+# -----------------------
+# Arabic normalization
+# -----------------------
+
+_DIACRITICS = re.compile(r"[\u064B-\u0652\u0670\u0640]")
+
+def normalize_arabic(text: str) -> str:
+    # 1. Remove diacritics
+    text = _DIACRITICS.sub("", text)
+
+    # 2. Normalize hamza variants
+    text = re.sub(r"[إأآ]", "ا", text)   # إ أ آ → ا
+    text = re.sub(r"[ؤئ]", "ء", text)    # ؤ ئ → ء
+
+    # 3. Normalize other orthographic variants
+    text = re.sub(r"ة", "ه", text)       # ة → ه
+    text = re.sub(r"ى", "ي", text)       # ى → ي
+
+    return text
+
+# -----------------------
+# Loader
+# -----------------------
+_MODEL = None
+
+def load_ner(
+    repo_id="Muhsabrys/AMWAL-ner-arabic",
+    local_path=None,
+):
+    """
+    Load AMWAL NER:
+    - from local_path (development / testing)
+    - or from Hugging Face (default)
+    """
+    global _MODEL
+
+    if _MODEL is None:
+        if local_path is not None:
+            model_path = os.path.join(local_path, "spacy_model", "model-best")
+        else:
+            path = snapshot_download(repo_id=repo_id)
+            model_path = os.path.join(path, "spacy_model", "model-best")
+
+        _MODEL = spacy.load(model_path)
+
+    def ner(text: str):
+        raw = text
+        text_norm = normalize_arabic(text)
+        doc = _MODEL(text_norm)
+
+        return {
+            "raw_text": raw,
+            "normalized_text": text_norm,
+            "entities": [
+                {
+                    "text": ent.text,
+                    "label": ent.label_,
+                    "start": ent.start_char,
+                    "end": ent.end_char,
+                }
+                for ent in doc.ents
+            ],
+        }
+
+    return ner

readme.md

@@ -0,0 +1,7 @@
+from amwal import load_ner
+
+ner = load_ner()
+
+text = "أعلن صندوق قطر السيادي عن استثمار بقيمة 500 مليون دولار أمريكي في سندات حكومية يابانية مقومة بالين في طوكيو."
+out = ner(text)
+print(out)

requirements.txt

@@ -0,0 +1,2 @@
+spacy>=3.7.0
+huggingface_hub>=0.20.0

spacy_model/model-best/config.cfg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c5db8dda5151ec3ab7983258bffcd45c70442618fc1dec41edfcca1f924f7da
+size 2722

spacy_model/model-best/meta.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:687cf15fbd3c932992f5b2c23dab124b33c54d9944df47bcd9238ea1068ca76d
+size 3271

spacy_model/model-best/ner/cfg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7172edadafba9f472e9ac0f2660eec04b6405e471be9e20267b79c67288d22d
+size 221

spacy_model/model-best/ner/model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc12d1c22c89822ecf06fdc9c456589f1f85bbc749cca862fafa95595efde0db
+size 189277

spacy_model/model-best/ner/moves

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f38206131ebb4afb533f4d694beb30e0983537efed33f065b64acfc582e81edd
+size 1532

spacy_model/model-best/pipeline.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6a1276222b80fe61d3023384a4019ccf2e940c1565ab9ad523efe756a858125
+size 1492

spacy_model/model-best/tok2vec/cfg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8a5a26e3056eb6fb06deeb3dbccfd88ae74900200c98c70b5966bbb7ec9d4de
+size 4

spacy_model/model-best/tok2vec/model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efa61cb31c443234eb772adc66a6803f84de2db5bf5be192a688f8a48c5cfdac
+size 34126801

spacy_model/model-best/tokenizer

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:673e9a42da7a084a0456720f4d2db1257598bcf0c8717e37a85aa98ecb086c44
+size 32716

spacy_model/model-best/vocab/key2row

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71
+size 1

spacy_model/model-best/vocab/lookups.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71
+size 1

spacy_model/model-best/vocab/strings.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcf49e22d635a86b3a698595916eb888c23d0443599a372cd81a75d0c0b69ed3
+size 12443841

spacy_model/model-best/vocab/vectors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14772b683e726436d5948ad3fff2b43d036ef2ebbe3458aafed6004e05a40706
+size 128

spacy_model/model-best/vocab/vectors.cfg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff4359091952c8cd16f1f0482f5770fb82d1707368d5cca3c46aa501f552e3c5
+size 22