import gradio as gr import pandas as pd import json import os import time from pathlib import Path from itertools import combinations # ============================================================ # Configuration # ============================================================ # Use HF Spaces persistent storage if available, else local PERSISTENT_DIR = "/data" if os.path.exists("/data") else "." DATA_DIR = os.path.join(PERSISTENT_DIR, "data") ANNOTATIONS_DIR = os.path.join(PERSISTENT_DIR, "annotations") OVERLAP_COUNT = 100 ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "admin123") os.makedirs(DATA_DIR, exist_ok=True) os.makedirs(ANNOTATIONS_DIR, exist_ok=True) # Copy bundled data to persistent storage on first run import shutil _local_data = os.path.join(os.path.dirname(__file__), "data") if _local_data != DATA_DIR and os.path.exists(_local_data): for fname in os.listdir(_local_data): dest = os.path.join(DATA_DIR, fname) if not os.path.exists(dest): shutil.copy2(os.path.join(_local_data, fname), dest) GRAMMATICAL_CLASSES = [ "noun", "verb", "adj", "adv", "pronoun", "demonstrative", "possessive", "conjunction", "interjection", "ideophone", "interrogative", "numeral", "copulative", "relative", "particle", "proper_name", "foreign", "abbreviation", "other" ] GLOSS_OPTIONS = [ "CL1", "CL2", "CL3", "CL4", "CL5", "CL6", "CL7", "CL8", "CL9", "CL10", "CL11", "CL14", "CL15", "CL16", "CL17", "CL18", "SP1sg", "SP2sg", "SP3sg", "SP1pl", "SP2pl", "SP3pl", "SC.CL1", "SC.CL2", "SC.CL3", "SC.CL4", "SC.CL5", "SC.CL6", "SC.CL7", "SC.CL8", "SC.CL9", "SC.CL10", "OM.1SG", "OM.2SG", "OM.1PL", "OM.CL1", "OM.CL2", "PRES", "PST", "FUT", "REC.PST", "REC.PST.LONG", "PST.CONT", "PROG", "PERF", "HAB", "COMPL", "TAM", "SP1sg.TAM", "CAUS", "APPL", "RECIP", "PASS", "STAT", "REV", "INTENS", "ITER", "INF", "FV", "FV.IMP.SG", "SUBJ.FV", "NEG", "NEG.PL", "HORT", "IMP.PL", "NMLZ", "DIM", "AUG", "FEM", "AG", "EPENTH", "root.IMBR", "REL", "DEM", "DEM.PROX", "DEM.DIST", "POSS", "PossConc", "PRON", "AbsPron", "COP", "CONJ", "IDEO", "INTJ", "INTERROG", "NUM", "ABBR", "ProperName", "Foreign", "TOP", "FOC", "DET", "AGR", ] TAG_OPTIONS = ["PREFIX", "ROOT", "SUFFIX", "INFIX"] # ============================================================ # Data Management # ============================================================ def get_word_list(): txt_path = os.path.join(DATA_DIR, "words.txt") xlsx_path = os.path.join(DATA_DIR, "words.xlsx") csv_path = os.path.join(DATA_DIR, "words.csv") if os.path.exists(txt_path): with open(txt_path, "r", encoding="utf-8") as f: return [line.strip() for line in f if line.strip()] elif os.path.exists(csv_path): df = pd.read_csv(csv_path, encoding='utf-8') if 'word' in df.columns: return df['word'].dropna().astype(str).tolist() return df.iloc[:, 0].dropna().astype(str).tolist() elif os.path.exists(xlsx_path): df = pd.read_excel(xlsx_path) if 'word' in df.columns: return df['word'].dropna().astype(str).tolist() return df.iloc[:, 0].dropna().astype(str).tolist() else: return [ "mutthu", "atthu", "kiruma", "kinaaruma", "orumiha", "anaakhula", "kharumeke", "orumela", "ekwaha", "ikwaha", "murutthu", "mirutthu", "niruma", "uruma", "mukaya", "vakaya", "okaya", "ukhuru", "ntoko", "makoho", "orumana", "orumiwa", "kinaavara", "anavara", "ekumi", ] * 44 def get_word_assignments(): """Load word assignment metadata (annotator column from CSV). Returns a list of annotator assignments per word index, or None if not available.""" meta_path = os.path.join(DATA_DIR, "words_metadata.json") if os.path.exists(meta_path): with open(meta_path, "r", encoding="utf-8") as f: metadata = json.load(f) return [item.get("annotator", "all") for item in metadata] return None def get_overlap_indices(all_words): """Get indices of overlap words (annotator='all' in metadata).""" assignments = get_word_assignments() if assignments: return [i for i, a in enumerate(assignments) if str(a).strip().lower() == "all"] # Fallback: use first OVERLAP_COUNT words return list(range(min(OVERLAP_COUNT, len(all_words)))) def get_annotations_path(annotator_id): return os.path.join(ANNOTATIONS_DIR, f"{annotator_id}.json") def load_annotations(annotator_id): path = get_annotations_path(annotator_id) if os.path.exists(path): with open(path, "r", encoding="utf-8") as f: return json.load(f) return {} def save_annotations(annotator_id, annotations): path = get_annotations_path(annotator_id) with open(path, "w", encoding="utf-8") as f: json.dump(annotations, f, ensure_ascii=False, indent=2) def get_all_annotators(): path = os.path.join(ANNOTATIONS_DIR, "_annotators.json") if os.path.exists(path): with open(path, "r", encoding="utf-8") as f: return json.load(f) return [] def save_annotators(annotators): path = os.path.join(ANNOTATIONS_DIR, "_annotators.json") with open(path, "w", encoding="utf-8") as f: json.dump(annotators, f) def get_annotator_word_list(annotator_id, all_words): """Get the list of word indices assigned to this annotator. - Words with annotator='all' in metadata go to everyone (overlap). - Words assigned to a specific annotator go only to them. - Words with no metadata assignment are distributed round-robin. """ annotators = get_all_annotators() if annotator_id not in annotators: annotators.append(annotator_id) save_annotators(annotators) assignments = get_word_assignments() if assignments: # Use metadata-based assignment overlap = [] assigned_to_me = [] unassigned = [] for i, assign in enumerate(assignments): assign_lower = str(assign).strip().lower() if assign_lower == "all": overlap.append(i) elif assign_lower == annotator_id.lower(): assigned_to_me.append(i) elif assign_lower in ("", "nan", "none"): unassigned.append(i) # else: assigned to another annotator, skip # Distribute unassigned words round-robin if unassigned and len(annotators) > 0: idx = annotators.index(annotator_id) my_unassigned = [unassigned[i] for i in range(len(unassigned)) if i % len(annotators) == idx] else: my_unassigned = [] return overlap + assigned_to_me + my_unassigned else: # Fallback: old behavior with OVERLAP_COUNT shared = list(range(min(OVERLAP_COUNT, len(all_words)))) remaining_count = len(all_words) - OVERLAP_COUNT if len(annotators) <= 1 or remaining_count <= 0: return list(range(len(all_words))) idx = annotators.index(annotator_id) assigned_remaining = [OVERLAP_COUNT + i for i in range(remaining_count) if i % len(annotators) == idx] return shared + assigned_remaining # ============================================================ # API Functions # ============================================================ def get_word_metadata(): """Load word metadata if available.""" meta_path = os.path.join(DATA_DIR, "words_metadata.json") if os.path.exists(meta_path): with open(meta_path, "r", encoding="utf-8") as f: return json.load(f) return None def get_valid_annotators(): """Get the list of valid annotator IDs from the CSV metadata.""" metadata = get_word_metadata() if metadata: annotators = set() for item in metadata: val = str(item.get("annotator", "")).strip().lower() if val and val not in ("all", "nan", "none", ""): annotators.add(val) return sorted(annotators) # Fallback: all registered annotators return get_all_annotators() def _get_word_id(global_idx, metadata=None): """Get the word ID from metadata, falling back to the global index.""" if metadata and global_idx < len(metadata): return str(metadata[global_idx].get("id", global_idx)) return str(global_idx) def api_get_word(annotator_id, word_idx): all_words = get_word_list() # Validate annotator valid = get_valid_annotators() if valid and annotator_id.lower() not in [v.lower() for v in valid]: return json.dumps({"error": f"Anotador '{annotator_id}' não encontrado. Válidos: {', '.join(valid)}"}) assigned = get_annotator_word_list(annotator_id, all_words) annotations = load_annotations(annotator_id) word_idx = int(word_idx) if word_idx >= len(assigned): word_idx = len(assigned) - 1 if word_idx < 0: word_idx = 0 global_idx = assigned[word_idx] word = all_words[global_idx] # Get word_id from metadata metadata = get_word_metadata() word_id = str(global_idx) # fallback if metadata and global_idx < len(metadata): word_id = str(metadata[global_idx].get("id", global_idx)) done = sum(1 for idx in assigned if _get_word_id(idx, metadata) in annotations) total = len(assigned) # Overlap based on metadata "all" assignment overlap_indices = get_overlap_indices(all_words) overlap_done = sum(1 for idx in overlap_indices if _get_word_id(idx, metadata) in annotations) overlap_total = len(overlap_indices) is_overlap = global_idx in overlap_indices existing = annotations.get(word_id, None) # Get sample text from metadata sample_text = "" if metadata and global_idx < len(metadata): sample_text = metadata[global_idx].get("sample_text", "") return json.dumps({ "word": word, "word_idx": word_idx, "global_idx": global_idx, "word_id": word_id, "total": total, "done": done, "overlap_done": overlap_done, "overlap_total": overlap_total, "is_overlap": is_overlap, "existing": existing, "sample_text": sample_text, }) def api_save_annotation(annotator_id, global_idx, morphemes_json, lemma, gram_class, comment, traducao=""): # Extract duration if appended duration_seconds = 0 if '|dur:' in traducao: parts = traducao.rsplit('|dur:', 1) traducao = parts[0] try: duration_seconds = int(parts[1]) except (ValueError, IndexError): pass data = json.loads(morphemes_json) morphemes = [m["text"] for m in data] tags = [m["tag"] for m in data] glosses = [m["gloss"] for m in data] if not morphemes: return json.dumps({"error": "Segmente a palavra primeiro"}) # Validation: all morphemes must have a tag; gloss required except for ROOT for i, (t, g) in enumerate(zip(tags, glosses)): if not t: return json.dumps({"error": f"Morfema '{morphemes[i]}' sem Tag atribuída"}) if not g and t.upper() != 'ROOT': return json.dumps({"error": f"Morfema '{morphemes[i]}' sem Glosa atribuída"}) if not lemma.strip(): return json.dumps({"error": "Preencha o campo Lemma"}) if not gram_class: return json.dumps({"error": "Selecione a Classe Gramatical"}) if not traducao.strip(): return json.dumps({"error": "Preencha o campo Tradução"}) annotation = { "word_id": str(global_idx), "morphemes": morphemes, "tags": tags, "glosses": glosses, "lemma": lemma.strip(), "grammatical_class": gram_class, "comment": comment.strip(), "traducao": traducao.strip(), "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), "gloss_complete": "-".join(glosses), "duration_seconds": duration_seconds, } annotations = load_annotations(annotator_id) annotations[str(global_idx)] = annotation save_annotations(annotator_id, annotations) return json.dumps({"success": True}) def api_next_unannotated(annotator_id, current_idx): all_words = get_word_list() assigned = get_annotator_word_list(annotator_id, all_words) annotations = load_annotations(annotator_id) metadata = get_word_metadata() current_idx = int(current_idx) for i in range(current_idx + 1, len(assigned)): word_id = _get_word_id(assigned[i], metadata) if word_id not in annotations: return json.dumps({"word_idx": i}) return json.dumps({"word_idx": current_idx}) def export_annotations(annotator_id): if not annotator_id: return None annotations = load_annotations(annotator_id) all_words = get_word_list() metadata = get_word_metadata() # Build id-to-index map id_to_idx = {} if metadata: for i, m in enumerate(metadata): id_to_idx[str(m.get("id", i))] = i rows = [] for word_id, ann in sorted(annotations.items(), key=lambda x: int(x[0]) if x[0].isdigit() else 0): idx = id_to_idx.get(word_id, int(word_id) if word_id.isdigit() else -1) word = all_words[idx] if 0 <= idx < len(all_words) else ann.get("word", "?") row = {"ID_Palavra": word_id, "Palavra": word, "Lemma": ann.get("lemma", "")} for i, (m, t, g) in enumerate(zip(ann.get("morphemes",[]), ann.get("tags",[]), ann.get("glosses",[])), 1): row[f"Morfema_{i}"] = m; row[f"Tag_{i}"] = t; row[f"Glosa_{i}"] = g row["Glosa_Completa"] = ann.get("gloss_complete", "") row["Classe_Gramatical"] = ann.get("grammatical_class", "") row["Traducao"] = ann.get("traducao", "") row["Anotador"] = annotator_id row["Notas"] = ann.get("comment", "") row["Duracao_Segundos"] = ann.get("duration_seconds", 0) rows.append(row) if not rows: return None df = pd.DataFrame(rows) export_path = os.path.join(ANNOTATIONS_DIR, f"export_{annotator_id}.xlsx") df.to_excel(export_path, index=False) return export_path def upload_words(file): if file is None: return "Nenhum ficheiro selecionado" fname = file.name if hasattr(file, 'name') else file if fname.endswith('.csv'): df = pd.read_csv(fname, encoding='utf-8') if 'word' in df.columns: words = df['word'].dropna().astype(str).tolist() # Save full metadata for reference df.to_json(os.path.join(DATA_DIR, "words_metadata.json"), orient='records', force_ascii=False) else: words = df.iloc[:, 0].dropna().astype(str).tolist() elif fname.endswith('.xlsx'): df = pd.read_excel(fname) if 'word' in df.columns: words = df['word'].dropna().astype(str).tolist() df.to_json(os.path.join(DATA_DIR, "words_metadata.json"), orient='records', force_ascii=False) else: words = df.iloc[:, 0].dropna().astype(str).tolist() elif fname.endswith('.txt'): with open(fname, 'r', encoding='utf-8') as f: words = [line.strip() for line in f if line.strip()] else: return "⚠️ Use .txt, .csv ou .xlsx" out_path = os.path.join(DATA_DIR, "words.txt") with open(out_path, "w", encoding="utf-8") as f: f.write("\n".join(words)) return f"✅ {len(words)} palavras carregadas!" # ============================================================ # Admin Functions # ============================================================ def api_admin_login(password): """Validate admin password.""" return json.dumps({"success": password == ADMIN_PASSWORD}) def compute_cohens_kappa(): """Compute Cohen's Kappa for inter-annotator agreement on overlap words (annotator='all').""" all_words = get_word_list() annotators = get_all_annotators() if len(annotators) < 2: return json.dumps({"error": "Necessário pelo menos 2 anotadores", "annotators": annotators}) overlap_indices = get_overlap_indices(all_words) overlap_count = len(overlap_indices) results = {"pairs": [], "annotators": annotators, "overlap_count": overlap_count} # Load all annotations for overlap words ann_data = {} for aid in annotators: anns = load_annotations(aid) ann_data[aid] = anns # For each pair of annotators, compute Kappa for a1, a2 in combinations(annotators, 2): labels_a1 = [] labels_a2 = [] common_count = 0 for idx in overlap_indices: idx_str = str(idx) if idx_str in ann_data[a1] and idx_str in ann_data[a2]: common_count += 1 gloss1 = ann_data[a1][idx_str].get("gloss_complete", "") gloss2 = ann_data[a2][idx_str].get("gloss_complete", "") labels_a1.append(gloss1) labels_a2.append(gloss2) if common_count < 2: results["pairs"].append({ "annotator1": a1, "annotator2": a2, "kappa": None, "common": common_count, "agreement_pct": None, "message": f"Poucos itens em comum ({common_count})" }) continue # Compute Cohen's Kappa agree = sum(1 for l1, l2 in zip(labels_a1, labels_a2) if l1 == l2) po = agree / common_count all_labels = set(labels_a1 + labels_a2) pe = 0.0 for label in all_labels: p1 = labels_a1.count(label) / common_count p2 = labels_a2.count(label) / common_count pe += p1 * p2 if pe == 1.0: kappa = 1.0 if po == 1.0 else 0.0 else: kappa = (po - pe) / (1 - pe) results["pairs"].append({ "annotator1": a1, "annotator2": a2, "kappa": round(kappa, 4), "common": common_count, "agreement_pct": round(po * 100, 1), "po": round(po, 4), "pe": round(pe, 4) }) # Per-level agreement (lemma, root, segmentation, gloss, grammatical class) for pair in results["pairs"]: if pair["kappa"] is None: continue a1, a2 = pair["annotator1"], pair["annotator2"] lemma_agree = 0 root_agree = 0 seg_agree = 0 gloss_agree = 0 gram_agree = 0 total = 0 for idx in overlap_indices: idx_str = str(idx) if idx_str in ann_data[a1] and idx_str in ann_data[a2]: total += 1 d1 = ann_data[a1][idx_str] d2 = ann_data[a2][idx_str] # Lemma agreement if d1.get("lemma", "").strip().lower() == d2.get("lemma", "").strip().lower(): lemma_agree += 1 # Root agreement (morpheme tagged ROOT) root1 = next((m for m, t in zip(d1.get("morphemes", []), d1.get("tags", [])) if t.upper() == "ROOT"), "") root2 = next((m for m, t in zip(d2.get("morphemes", []), d2.get("tags", [])) if t.upper() == "ROOT"), "") if root1.lower() == root2.lower(): root_agree += 1 # Segmentation agreement (full morpheme list) if d1.get("morphemes", []) == d2.get("morphemes", []): seg_agree += 1 # Gloss complete agreement if d1.get("gloss_complete", "").strip() == d2.get("gloss_complete", "").strip(): gloss_agree += 1 # Grammatical class agreement if d1.get("grammatical_class", "").strip().lower() == d2.get("grammatical_class", "").strip().lower(): gram_agree += 1 if total > 0: pair["lemma_agreement"] = round(lemma_agree / total * 100, 1) pair["root_agreement"] = round(root_agree / total * 100, 1) pair["segmentation_agreement"] = round(seg_agree / total * 100, 1) pair["gloss_agreement"] = round(gloss_agree / total * 100, 1) pair["gram_class_agreement"] = round(gram_agree / total * 100, 1) # Compute "all" row — agreement where ALL annotators annotated the same word valid_pairs = [p for p in results["pairs"] if p["kappa"] is not None] if len(annotators) >= 2: # Find overlap words annotated by ALL annotators all_common = 0 all_lemma = 0 all_root = 0 all_seg = 0 all_gloss = 0 all_gram = 0 for idx in overlap_indices: idx_str = str(idx) annotations_for_word = [ann_data[a][idx_str] for a in annotators if idx_str in ann_data[a]] if len(annotations_for_word) < 2: continue all_common += 1 # All agree = all values are the same lemmas = [d.get("lemma", "").strip().lower() for d in annotations_for_word] if len(set(lemmas)) == 1: all_lemma += 1 roots = [next((m for m, t in zip(d.get("morphemes", []), d.get("tags", [])) if t.upper() == "ROOT"), "") for d in annotations_for_word] if len(set(r.lower() for r in roots)) == 1: all_root += 1 segs = [tuple(d.get("morphemes", [])) for d in annotations_for_word] if len(set(segs)) == 1: all_seg += 1 glosses = [d.get("gloss_complete", "").strip() for d in annotations_for_word] if len(set(glosses)) == 1: all_gloss += 1 grams = [d.get("grammatical_class", "").strip().lower() for d in annotations_for_word] if len(set(grams)) == 1: all_gram += 1 all_row = { "annotator1": " / ".join(annotators), "annotator2": "(todos)", "common": all_common, "kappa": round(sum(p["kappa"] for p in valid_pairs) / len(valid_pairs), 4) if valid_pairs else None, "agreement_pct": round(sum(p.get("agreement_pct", 0) for p in valid_pairs) / len(valid_pairs), 1) if valid_pairs else None, } if all_common > 0: all_row["lemma_agreement"] = round(all_lemma / all_common * 100, 1) all_row["root_agreement"] = round(all_root / all_common * 100, 1) all_row["segmentation_agreement"] = round(all_seg / all_common * 100, 1) all_row["gloss_agreement"] = round(all_gloss / all_common * 100, 1) all_row["gram_class_agreement"] = round(all_gram / all_common * 100, 1) results["summary"] = all_row return json.dumps(results) def api_admin_delete_annotations(annotator_id, indices_json): """Delete specific annotations for an annotator. indices_json is a JSON array of word indices.""" try: indices = json.loads(indices_json) except: return json.dumps({"error": "Formato inválido"}) if not indices: return json.dumps({"error": "Nenhuma anotação selecionada"}) annotations = load_annotations(annotator_id) deleted = 0 for idx in indices: idx_str = str(idx) if idx_str in annotations: del annotations[idx_str] deleted += 1 save_annotations(annotator_id, annotations) return json.dumps({"success": True, "deleted": deleted, "message": f"✅ {deleted} anotação(ões) removida(s) de '{annotator_id}'"}) def save_annotations(annotator_id, annotations): """Save full annotations dict for an annotator.""" path = os.path.join(ANNOTATIONS_DIR, f"{annotator_id}.json") with open(path, "w", encoding="utf-8") as f: json.dump(annotations, f, ensure_ascii=False, indent=2) def api_admin_delete_words(indices_json): """Delete words from the word list by their global indices.""" try: indices = json.loads(indices_json) except: return json.dumps({"error": "Formato inválido"}) if not indices: return json.dumps({"error": "Nenhuma palavra selecionada"}) indices_set = set(int(i) for i in indices) # Load metadata and filter out deleted words meta_path = os.path.join(DATA_DIR, "words_metadata.json") words_path = os.path.join(DATA_DIR, "words.txt") if os.path.exists(meta_path): with open(meta_path, "r", encoding="utf-8") as f: metadata = json.load(f) metadata = [m for i, m in enumerate(metadata) if i not in indices_set] with open(meta_path, "w", encoding="utf-8") as f: json.dump(metadata, f, ensure_ascii=False) words = [m["word"] for m in metadata] else: words = get_word_list() words = [w for i, w in enumerate(words) if i not in indices_set] with open(words_path, "w", encoding="utf-8") as f: f.write("\n".join(words)) return json.dumps({"success": True, "message": f"✅ {len(indices_set)} palavra(s) removida(s). Total: {len(words)}"}) def api_admin_get_valid_annotators(): """Return valid annotators from CSV.""" return json.dumps({"annotators": get_valid_annotators()}) def api_admin_export_all(): """Export all annotators' annotations into a single Excel file.""" annotators = get_all_annotators() all_words = get_word_list() metadata = get_word_metadata() id_to_idx = {} if metadata: for i, m in enumerate(metadata): id_to_idx[str(m.get("id", i))] = i all_rows = [] for aid in annotators: annotations = load_annotations(aid) for word_id, ann in sorted(annotations.items(), key=lambda x: int(x[0]) if x[0].isdigit() else 0): idx = id_to_idx.get(word_id, int(word_id) if word_id.isdigit() else -1) word = all_words[idx] if 0 <= idx < len(all_words) else "?" row = {"ID_Palavra": word_id, "Palavra": word, "Anotador": aid, "Lemma": ann.get("lemma", "")} for i, (m, t, g) in enumerate(zip(ann.get("morphemes",[]), ann.get("tags",[]), ann.get("glosses",[])), 1): row[f"Morfema_{i}"] = m; row[f"Tag_{i}"] = t; row[f"Glosa_{i}"] = g row["Glosa_Completa"] = ann.get("gloss_complete", "") row["Classe_Gramatical"] = ann.get("grammatical_class", "") row["Traducao"] = ann.get("traducao", "") row["Notas"] = ann.get("comment", "") row["Duracao_Segundos"] = ann.get("duration_seconds", 0) row["Timestamp"] = ann.get("timestamp", "") all_rows.append(row) if not all_rows: return None df = pd.DataFrame(all_rows) export_path = os.path.join(ANNOTATIONS_DIR, "export_all_annotations.xlsx") df.to_excel(export_path, index=False) return export_path def api_admin_stats(): """Get annotation statistics for admin panel.""" annotators = get_all_annotators() all_words = get_word_list() overlap_indices = get_overlap_indices(all_words) overlap_count = len(overlap_indices) stats = [] for aid in annotators: anns = load_annotations(aid) assigned = get_annotator_word_list(aid, all_words) done = sum(1 for idx in assigned if str(idx) in anns) overlap_done = sum(1 for idx in overlap_indices if str(idx) in anns) stats.append({ "annotator": aid, "total_assigned": len(assigned), "done": done, "overlap_done": overlap_done, "overlap_total": overlap_count, "progress_pct": round(done / len(assigned) * 100, 1) if assigned else 0 }) # Build words list with annotator assignment for word management metadata = [] meta_path = os.path.join(DATA_DIR, "words_metadata.json") if os.path.exists(meta_path): with open(meta_path, "r", encoding="utf-8") as f: metadata = json.load(f) words_info = [] for i, w in enumerate(all_words): ann_info = metadata[i].get("annotator", "") if i < len(metadata) else "" words_info.append({"word": w, "annotator": ann_info}) return json.dumps({"stats": stats, "total_words": len(all_words), "total_annotators": len(annotators), "words": words_info}) def api_admin_review(annotator_id): """Get all annotations for a specific annotator for review.""" all_words = get_word_list() anns = load_annotations(annotator_id) if not anns: return json.dumps({"error": f"Nenhuma anotação encontrada para '{annotator_id}'", "annotations": []}) rows = [] for idx_str, ann in sorted(anns.items(), key=lambda x: int(x[0])): idx = int(idx_str) word = all_words[idx] if idx < len(all_words) else "?" rows.append({ "idx": idx, "word": word, "morphemes": ann.get("morphemes", []), "tags": ann.get("tags", []), "glosses": ann.get("glosses", []), "gloss_complete": ann.get("gloss_complete", ""), "lemma": ann.get("lemma", ""), "grammatical_class": ann.get("grammatical_class", ""), "traducao": ann.get("traducao", ""), "comment": ann.get("comment", ""), "timestamp": ann.get("timestamp", "") }) return json.dumps({"annotator": annotator_id, "count": len(rows), "annotations": rows}) # ============================================================ # HTML Frontend # ============================================================ GUIDELINE_HTML = """

📖 Guia de Anotação Morfológica — Emakhuwa

Manual do Anotador | Felermino Ali | Versão 3.0 | Junho 2026

🖱️ Interação com a Ferramenta

⚠️ Campos Obrigatórios

🔗 Overlap (Acordo Inter-Anotadores)

Palavras marcadas como "OVERLAP" (com annotator=all no ficheiro) são anotadas por todos os anotadores para calcular o acordo (Cohen's Kappa). Anote-as com especial cuidado e precisão ≥ 98%.

1. Objetivo da Anotação

2. Conceitos Fundamentais

MorfemaUnidade simbólica mínima gramaticalmente significativa. Ex: mu- (CL1), -tthu (pessoa)
Raiz (Root)Núcleo semântico sem afixos. Verbal: -rum- (falar), -ven- (vir). Nominal: -tthu (pessoa)
Tema/StemNominal: raiz sem prefixos de classe. Verbal: raiz + extensões + vogal final (-ruma, -rumela)
Prefixo de ClassePrefixo básico (BPre) + pré-prefixo (NPrePre) quando aplicável

3. Como Segmentar uma Palavra

  1. Identifique a raiz (ROOT) — o núcleo semântico
  2. Identifique prefixos (à esquerda) — classe, sujeito, tempo, objeto, negação
  3. Identifique sufixos (à direita) — extensões verbais, vogais finais
  4. Verifique se há infixos dentro da raiz (ex: imbricação)

4. Sistema de Tags

Classes Nominais

ClassePrefixoSemânticaExemplo
CL1/CL2mu-/a-Humanos (sg/pl)mutthu/atthu
CL3/CL4mu-/mi-Árvores/plantasmurutthu/mirutthu
CL5/CL6e-,ni-/ma-Aumentativos, líquidosenimu/makoho
CL7/CL8e-/i-Coisas/instrumentosekwaha/ikwaha
CL9/CL10N-/N-Animais, empréstimosntoko/ntoko
CL11o-Objetos longos/finosothukha
CL14u-Abstratosukhuru
CL15/INFo-Infinitivooruma
CL16/17/18va-/o-/mu-Locativosvakaya/okaya/mukaya

Prefixos de Sujeito (SP)

PessoaTagFormaExemplo
1ª sg[SP1sg]ki-kiruma (eu falo)
2ª sg[SP2sg]u-uruma (tu falas)
3ª sg[SP3sg]a-/o-aruma (ele fala)
1ª pl[SP1pl]ni-niruma (nós falamos)
2ª pl[SP2pl]mu-muruma (vós falais)
3ª pl[SP3pl]a-aruma (eles falam)

Marcadores de Objeto (OM)

PessoaTagForma
1ª sg[OM.1SG]ki-
2ª sg[OM.2SG]ku-
1ª pl[OM.1PL]ni-
CL1[OM.CL1]mu-
CL2[OM.CL2]a-

Tempo/Aspeto/Modo (TAM)

TagDescriçãoForma
[PRES]Presente/Futuro-naa-
[PST]Passado
[REC.PST]Passado recente-ho- / -ale
[REC.PST.LONG]Passado recente (vogal longa)-hoo- / -aale
[PST.CONT]Passado contínuo-aa-
[PROG]Progressivo-noo-
[PERF]Perfectivo/Perfeito
[HAB]Habitual
[FUT]Futuro

Extensões Verbais (Sufixos Derivacionais)

TagDescriçãoFormaExemplo
[CAUS]Causativo-iha/-eha-rumiha (fazer falar)
[APPL]Aplicativo-el-/-al--rumela (falar para)
[RECIP]Recíproco-an--rumana (falar um com outro)
[PASS]Passivo-iw-/-ew--rumiwa (ser falado)
[STAT]Estativo-ak-/-ek--rumeka (ser falável)
[REV]Reversivo-ul-/-ol-(desfazer ação)
[INTENS]Intensivo-exa/-axa(intensificar)
[ITER]Iterativo(ação repetida)

Raiz Verbal e Vogal Final

TagDescriçãoExemplo
[VRoot] / ROOTRaiz verbal-rum- (falar), -ven- (vir), -khul- (crescer)
[FV]Vogal final (indicativo)-a
[FV.IMP.SG]Vogal final imperativo sg-a
[SUBJ.FV]Vogal final subjuntivo-e

Outras Categorias Verbais

TagDescriçãoForma
[INF]Prefixo infinitivoo-
[NEG]Negaçãokha-
[NEG.PL]Negação pluralka-
[HORT]Hortativo (vamos...)ka-
[IMP.PL]Imperativo plural=ni (enclítico)

Tags Nominais Adicionais

TagDescrição
[NStem]Raiz/tema nominal
[AUG]Sufixo aumentativo
[DIM]Sufixo diminutivo (-ana)
[FEM]Sufixo feminino (-ana)
[LocSuf]Sufixo locativo
[AG]Sufixo agentivo (-i/-u)

Outras Categorias Gramaticais

TagDescrição
[AdjStem] / [AdjConc]Raiz/concordância adjetival
[Adv]Advérbio
[DEM] / [DEM.PROX] / [DEM.DIST]Demonstrativo (proximal/distal)
[POSS] / [PossConc]Possessivo/concordância
[PRON] / [AbsPron]Pronome/pronome absoluto
[REL]Marcador relativo
[COP]Cópula
[CONJ]Conjunção
[IDEO]Ideofone
[INTJ]Interjeição
[INTERROG]Interrogativo
[NUM]Numeral
[NMLZ]Nominalizador
[EPENTH]Epentético (-ij- em raízes curtas)
[root.IMBR]Raiz imbricada
[ABBR]Abreviatura
[ProperName]Nome próprio
[Foreign]Palavra estrangeira

5. Exemplos Anotados

PalavraSegmentaçãoTagsGlosasTradução
mutthumu-tthuPREFIX-ROOTCL1-pessoapessoa
atthua-tthuPREFIX-ROOTCL2-pessoapessoas
kirumaki-rum-aPREFIX-ROOT-SUFFIXSP1sg-falar-FVeu falo
kinaarumaki-naa-rum-aPREFIX-PREFIX-ROOT-SUFFIXSP1sg-PRES-falar-FVeu estou a falar
orumihao-rum-iha-aPREFIX-ROOT-SUFFIX-SUFFIXINF-falar-CAUS-FVfazer falar
anaakhulaa-naa-khul-aPREFIX-PREFIX-ROOT-SUFFIXSP3pl-PST.CONT-crescer-FVeles cresciam
kharumekekha-rum-ek-ePREFIX-ROOT-SUFFIX-SUFFIXNEG-falar-STAT-SUBJ.FVnão é falável
orumelao-rum-el-aPREFIX-ROOT-SUFFIX-SUFFIXINF-falar-APPL-FVfalar para
ekwahae-kwahaPREFIX-ROOTCL7-coisacoisa
ikwahai-kwahaPREFIX-ROOTCL8-coisacoisas

6. Glossário de Abreviaturas

Derivação:
NMLZ (Nominalizador)
CAUS (Causativo)
APPL (Aplicativo)
RECIP (Recíproco)
DIM (Diminutivo)
AUG (Aumentativo)
INTENS (Intensivo)
STAT (Estativo)
REV (Reversivo)
ITER (Iterativo)
AG (Agentivo)		 TAM:
PRES (Presente)
PST (Passado)
FUT (Futuro)
REC.PST (Passado recente)
PST.CONT (Passado contínuo)
PROG (Progressivo)
PERF (Perfectivo)
HAB (Habitual)
IND (Indicativo)
SUBJ (Subjuntivo)
IMP (Imperativo)
HORT (Hortativo)		 Pessoa:
1SG, 2SG, 3SG
1PL, 2PL, 3PL

Polaridade:
NEG (Negação)
FV (Vogal final)
REL (Relativo)
DEM (Demonstrativo)
TOP (Tópico)
FOC (Foco)

Voz:
PASS (Passiva)
ACT (Ativa)

7. Erros Comuns a Evitar

  1. Não deixar campos vazios: preencha sempre Tag, Glosa, Lemma, Classe e Tradução
  2. Não confundir ROOT com PREFIX: a raiz é o núcleo semântico
  3. Não inventar tags: use apenas as tags definidas neste guia
  4. Não usar tags genéricas (PREFIX, SUFFIX) quando a função é conhecida — prefira CL1, SP1sg, CAUS, etc.
  5. Não confundir prefixos homófonos: mu- pode ser CL1, CL3 ou CL18 — analise o contexto
  6. Não misturar a ordem: preencha morfemas da esquerda para a direita
  7. Não separar extensões fusionadas: se CAUS+APPL estão fusionados, trate como unidade
  8. Não consultar outros anotadores: trabalho deve ser independente

8. Regras Importantes

9. Referências

""" def build_html(): result = f"""

🏷️ Anotação Morfológica — Emakhuwa

Ferramenta de anotação morfológica interativa


🔐 Acesso Administrador
Carregando...
🔗 OVERLAP

⚙️ Painel de Administração

📊 Estatísticas dos Anotadores

📐 Acordo Inter-Anotadores (Cohen's Kappa)

Calcula o acordo entre pares de anotadores nas palavras de overlap (annotator=all).

📤 Carregar Palavras

📥 Exportar Anotações

Exporta todas as anotações de todos os anotadores num ficheiro Excel.

🔍 Revisar Anotações

Visualizar e gerir anotações de um anotador específico.

📋 Gerir Palavras

Pesquisar e apagar palavras da lista de anotação.

{GUIDELINE_HTML}

Editar Morfema

""" # Split into HTML (before ', '').replace('\n\n', '') return html_only, js_code # ============================================================ # Gradio App # ============================================================ html_content, js_code = build_html() head_script = "" def api_upload_words_from_path(file_path): """Handle upload via file path from admin panel.""" if not file_path: return json.dumps("Nenhum ficheiro selecionado") if file_path.endswith('.csv'): df = pd.read_csv(file_path, encoding='utf-8') if 'word' in df.columns: words = df['word'].dropna().astype(str).tolist() df.to_json(os.path.join(DATA_DIR, "words_metadata.json"), orient='records', force_ascii=False) else: words = df.iloc[:, 0].dropna().astype(str).tolist() elif file_path.endswith('.xlsx'): df = pd.read_excel(file_path) if 'word' in df.columns: words = df['word'].dropna().astype(str).tolist() df.to_json(os.path.join(DATA_DIR, "words_metadata.json"), orient='records', force_ascii=False) else: words = df.iloc[:, 0].dropna().astype(str).tolist() elif file_path.endswith('.txt'): with open(file_path, 'r', encoding='utf-8') as f: words = [line.strip() for line in f if line.strip()] else: return json.dumps("⚠️ Use .txt, .csv ou .xlsx") out_path = os.path.join(DATA_DIR, "words.txt") with open(out_path, "w", encoding="utf-8") as f: f.write("\n".join(words)) return json.dumps(f"✅ {len(words)} palavras carregadas!") def api_admin_export_all_wrapper(): path = api_admin_export_all() if path: return json.dumps({"path": path}) return json.dumps(None) with gr.Blocks(title="Emakhuwa Morphological Annotation", theme=gr.themes.Soft(), head=head_script) as demo: gr.HTML(html_content) # Hidden API components with gr.Column(visible=False): api_in1 = gr.Textbox() api_in2 = gr.Textbox() api_in3 = gr.Textbox() api_in4 = gr.Textbox() api_in5 = gr.Textbox() api_in6 = gr.Textbox() api_in7 = gr.Textbox() api_out = gr.Textbox() get_word_btn = gr.Button("get_word") get_word_btn.click( fn=lambda a, b, *_: api_get_word(a, int(b)), inputs=[api_in1, api_in2], outputs=[api_out], api_name="get_word" ) save_btn = gr.Button("save") save_btn.click( fn=lambda a, b, c, d, e, f, g: api_save_annotation(a, int(b), c, d, e, f, g), inputs=[api_in1, api_in2, api_in3, api_in4, api_in5, api_in6, api_in7], outputs=[api_out], api_name="save_annotation" ) next_btn = gr.Button("next") next_btn.click( fn=lambda a, b, *_: api_next_unannotated(a, int(b)), inputs=[api_in1, api_in2], outputs=[api_out], api_name="next_unannotated" ) # Admin APIs admin_login_btn = gr.Button("admin_login") admin_login_btn.click( fn=lambda pw, *_: api_admin_login(pw), inputs=[api_in1], outputs=[api_out], api_name="admin_login" ) admin_stats_btn = gr.Button("admin_stats") admin_stats_btn.click( fn=lambda *_: api_admin_stats(), inputs=[api_in1], outputs=[api_out], api_name="admin_stats" ) compute_kappa_btn = gr.Button("compute_kappa") compute_kappa_btn.click( fn=lambda *_: compute_cohens_kappa(), inputs=[api_in1], outputs=[api_out], api_name="compute_kappa" ) upload_words_btn = gr.Button("upload_words") upload_words_btn.click( fn=lambda path, *_: api_upload_words_from_path(path), inputs=[api_in1], outputs=[api_out], api_name="upload_words" ) admin_export_btn = gr.Button("admin_export_all") admin_export_btn.click( fn=lambda *_: api_admin_export_all_wrapper(), inputs=[api_in1], outputs=[api_out], api_name="admin_export_all" ) admin_review_btn = gr.Button("admin_review") admin_review_btn.click( fn=lambda aid, *_: api_admin_review(aid), inputs=[api_in1], outputs=[api_out], api_name="admin_review" ) admin_delete_btn = gr.Button("admin_delete_annotations") admin_delete_btn.click( fn=lambda aid, indices, *_: api_admin_delete_annotations(aid, indices), inputs=[api_in1, api_in2], outputs=[api_out], api_name="admin_delete_annotations" ) admin_delete_words_btn = gr.Button("admin_delete_words") admin_delete_words_btn.click( fn=lambda indices, *_: api_admin_delete_words(indices), inputs=[api_in1], outputs=[api_out], api_name="admin_delete_words" ) demo.launch( allowed_paths=[ANNOTATIONS_DIR, DATA_DIR], server_name="0.0.0.0", server_port=7860, ssr_mode=False, show_api=True )