count all variants

measures/VocabularyAnalyser.py

@@ -1,7 +1,6 @@
 import pandas as pd
 import re
 import unicodedata
-import weakref
 
 # --- helper normalization ---
 def norm_txt(x: str) -> str:
@@ -17,17 +16,25 @@ class VocabularyAnalyser:
     def __init__(self, glossary_file: str):
         # Load glossary CSV (first column = base + variants, comma-separated)
         raw = pd.read_csv(glossary_file)
-        terms_col = raw.iloc[:, 0].astype(str).apply(norm_txt)
 
         gloss_list = []
-        for idx, cell in enumerate(terms_col, start=1):
-            items = [t.strip() for t in cell.split(",") if t.strip()]
-            if not items:
+        for idx, row in raw.iterrows():
+            forms = []
+            for cell in row:
+                if pd.isna(cell):
+                    continue
+                for chunk in str(cell).split(","):
+                    token = norm_txt(chunk)
+                    if token:
+                        forms.append(token)
+
+            if not forms:
                 continue
-            base = items[0]
-            for form in items:
+
+            base = forms[0]
+            for form in forms:
                 gloss_list.append({
-                    "row": idx,
+                    "row": idx + 1,
                     "base": base,
                     "form": form,
                     "len": len(form),