"""Given a list of list of analyses (one analysis per token), get features based on a given criteria. If no criteria is given, return atbtok and catib6 """ import re from typing import List import json from camel_tools.utils.dediac import dediac_ar from camel_tools.utils.charmap import CharMapper from camel_tools.utils.transliterate import Transliterator import pandas as pd FEATURES_LIST = ["pos", "prc3", "prc2", "prc1", "prc0", "enc0", "asp", "vox", "mod", "gen", "num", "stt", "cas", "per", "rat"] def feats_dict_to_string(feats_dict): # prc3=na|prc2=na|prc1=na|prc0=na|per=na|asp=na|vox=na|mod=na|gen=na|num=na|stt=na|cas=na|enc0=na|rat=na feats_str = json.dumps(feats_dict) return feats_str.replace('", "', "|").replace('": "', "=").replace('{"', '').replace('"}', '') def build_clitic_feats_dict(clitic_feat_list): assert len(clitic_feat_list) != 0, f'invalid clitic, {clitic_feat_list}' assert len(clitic_feat_list) == 1, f'multiple clitics matched when only one should, {clitic_feat_list}' clitic_feat_list[0]['deciding_feat'] final_clitic_feats = {k: v for k,v in clitic_feat_list[0].items() if k in FEATURES_LIST} final_clitic_feats['token_type'] = clitic_feat_list[0]['deciding_feat'].split(':')[0] return final_clitic_feats def get_clitic_feats(token, clitic_order, clitic_feats, stem_feats): mapper = CharMapper.builtin_mapper('ar2bw') transliterator = Transliterator(mapper) token = transliterator.transliterate(dediac_ar(token)) filtered_clitics = clitic_feats[(clitic_feats.clitic == token) & (clitic_feats.deciding_feat.str.startswith(clitic_order))] clitic_list = [f'{k}:{v}' for k, v in stem_feats.items() if k.startswith(clitic_order) and v not in ['0', 'na']] for feat_check in clitic_list: clitic_feat_list = filtered_clitics[filtered_clitics.deciding_feat == feat_check].to_dict('records') if clitic_feat_list: return build_clitic_feats_dict(clitic_feat_list) assert False, f"clitic '{token}' does not exist in clitics list. Stem features: {stem_feats}" def get_stem_feats(word_analysis): return {feat: word_analysis[feat] for feat in FEATURES_LIST} def get_clitic_order(token): if token.endswith('+'): clitic_order = 'prc' elif token.startswith('+'): clitic_order = 'enc' return clitic_order def is_clitic(token): return (token.startswith('+') or token.endswith('+')) and not re.match(r'^\++$', token) def empty_clitic_feats_from_baseword(stem_feats): clitic_type_list = ["prc3", "prc2", "prc1", "prc0", "enc0"] for clitic_type in clitic_type_list: if stem_feats[clitic_type] not in ['0', 'na', 'Al_det']: stem_feats[clitic_type] = '0' return stem_feats def r13_fixes(token, stem_feats): # handling edge cases where li comes from variations of wa li>n (i.e. li>nhu, li>nanY) li_feats = {'pos': 'conj_sub', 'prc3': '0', 'prc2': 'wa_conj', 'prc1': '0', 'prc0': 'na', 'enc0': '3ms_pron', 'asp': 'na', 'vox': 'na', 'mod': 'na', 'gen': 'na', 'num': 'na', 'stt': 'na', 'cas': 'na', 'per': 'na', 'rat': 'na'} if {k for k, _ in stem_feats.items() ^ li_feats.items()} == {'enc0'}: stem_feats['prc1'] = 'li_conj' # handling edge cases where li comes from variations of li>n (i.e. li>nhu, li>nanY) li_feats = {'pos': 'conj_sub', 'prc3': '0', 'prc2': '0', 'prc1': '0', 'prc0': 'na', 'enc0': '0', 'asp': 'na', 'vox': 'na', 'mod': 'na', 'gen': 'na', 'num': 'na', 'stt': 'na', 'cas': 'na', 'per': 'na', 'rat': 'na'} if {k for k, _ in stem_feats.items() ^ li_feats.items()} == {'enc0'}: stem_feats['prc1'] = 'li_conj' # handling edge cases where li comes from li>n if token == 'لِ+' and stem_feats == {'pos': 'conj_sub', 'prc3': '0', 'prc2': '0', 'prc1': '0', 'prc0': 'na', 'enc0': '0', 'asp': 'na', 'vox': 'na', 'mod': 'na', 'gen': 'na', 'num': 'na', 'stt': 'na', 'cas': 'na', 'per': 'na', 'rat': 'na'}: stem_feats['prc1'] = 'li_conj' # handling edge cases where li comes from wa li>n if token == 'لِ+' and stem_feats == {'pos': 'conj_sub', 'prc3': '0', 'prc2': 'wa_conj', 'prc1': '0', 'prc0': 'na', 'enc0': '3ms_pron', 'asp': 'na', 'vox': 'na', 'mod': 'na', 'gen': 'na', 'num': 'na', 'stt': 'na', 'cas': 'na', 'per': 'na', 'rat': 'na'}: stem_feats['prc1'] = 'li_conj' # handling edge cases where mA comes from qlmA if token == '+ما' and stem_feats == {'pos': 'conj', 'prc3': '0', 'prc2': '0', 'prc1': '0', 'prc0': 'na', 'enc0': '0', 'asp': 'na', 'vox': 'na', 'mod': 'na', 'gen': 'na', 'num': 'na', 'stt': 'na', 'cas': 'na', 'per': 'na', 'rat': 'n'}: stem_feats['enc0'] = 'mA_sub' # handling an edge case where li comes from likY # added li_conj to clitic_feats.csv if token == 'لِ+' and stem_feats == {'pos': 'conj', 'prc3': '0', 'prc2': '0', 'prc1': '0', 'prc0': 'na', 'enc0': '0', 'asp': 'na', 'vox': 'na', 'mod': 'na', 'gen': 'na', 'num': 'na', 'stt': 'na', 'cas': 'na', 'per': 'na', 'rat': 'na'}: stem_feats['prc1'] = 'li_conj' def add_remaining_features(tokens_df, stem_feats, clitic_feats): existing_clitics = ['prc0'] clitic_feats_list = [] for _, row in tokens_df.iterrows(): token = row['token'] if not is_clitic(token): baseword_feats_dict = empty_clitic_feats_from_baseword(dict(stem_feats)) baseword_feats_dict['token_type'] = 'baseword' clitic_feats_list.append(baseword_feats_dict) else: clitic_order = get_clitic_order(token) # handling an edge case where lA is negative if token == 'لِ+' and \ (stem_feats == {'pos': 'conj_sub', 'prc3': '0', 'prc2': '0', 'prc1': '0', 'prc0': 'na', 'enc0': 'lA_neg', 'asp': 'na', 'vox': 'na', 'mod': 'na', 'gen': 'na', 'num': 'na', 'stt': 'na', 'cas': 'na', 'per': 'na', 'rat': 'na'} or stem_feats == {'pos': 'conj_sub', 'prc3': '0', 'prc2': 'fa_conj', 'prc1': '0', 'prc0': 'na', 'enc0': 'lA_neg', 'asp': 'na', 'vox': 'na', 'mod': 'na', 'gen': 'na', 'num': 'na', 'stt': 'na', 'cas': 'na', 'per': 'na', 'rat': 'na'} or stem_feats == {'pos': 'conj_sub', 'prc3': '0', 'prc2': 'wa_part', 'prc1': '0', 'prc0': 'na', 'enc0': 'lA_neg', 'asp': 'na', 'vox': 'na', 'mod': 'na', 'gen': 'na', 'num': 'na', 'stt': 'na', 'cas': 'na', 'per': 'na', 'rat': 'na'} ): stem_feats['prc1'] = 'li_prep' r13_fixes(token, stem_feats) clitic_feats_list.append(get_clitic_feats(token.replace('+', ''), clitic_order, clitic_feats, stem_feats)) feats_df = pd.DataFrame(clitic_feats_list) assert tokens_df.shape[0] == feats_df.shape[0], f'token-feature mismatch!,\ntokens: \n{tokens_df},\n\n features: \n{feats_df}' return pd.concat([tokens_df, feats_df], axis=1) def get_lemmas(lemma, tokens): lemmas = [] for token in tokens: if '+' in token: lemmas.append(token) else: lemmas.append(lemma) return lemmas def get_main_features_df(word_analysis): # if there are no clitics if '+' not in word_analysis['catib6']: tokens = [word_analysis['atbtok']] catib6 = [word_analysis['catib6']] ud = [word_analysis['ud']] lemmas = [word_analysis['lex']] else: tokens = word_analysis['atbtok'].split('_') catib6 = word_analysis['catib6'].split('+') ud = word_analysis['ud'].split('+') lemmas = get_lemmas(word_analysis['lex'], tokens) if len(catib6) < len(tokens): print(tokens) print(catib6) catib6.append("NOM") ud.append("NOUN") return pd.DataFrame({'token': tokens, 'catib6': catib6, 'ud': ud, 'lemma': lemmas}) elif len(tokens) < len(catib6): return pd.DataFrame({'token': tokens, 'catib6': catib6[:len(tokens)], 'ud': ud[:len(tokens)], 'lemma': lemmas}) return pd.DataFrame({'token': tokens, 'catib6': catib6, 'ud': ud, 'lemma': lemmas}) def get_word_features_df(word_analysis, clitic_feats): """if a word is composed of multiple tokens, return them all. otherwise, just return the word and catib6 tag Args: word_analysis (dict): analysis generated from a cameltools disambiguator Returns: List[tuple]: a list of one or more tokens """ # gets forms, tokens, feats, ... main_feats = get_main_features_df(word_analysis) stem_feats = get_stem_feats(word_analysis) return add_remaining_features(main_feats, stem_feats, clitic_feats) def join_feats(word_feats_df, tagset): word_features = { 'tokens': list(word_feats_df['token']), 'pos_tags': list(word_feats_df[tagset]), 'lemmas': list(word_feats_df['lemma']), } word_feats_df.drop(['token', tagset, 'lemma'], axis=1, inplace=True) feats = word_feats_df.to_dict('records') word_features['feats'] = [feats_dict_to_string(row) for row in feats] return word_features def update_sentence_features(sentence_features, word_features): sentence_features['tokens'] += word_features['tokens'] sentence_features['lemmas'] += word_features['lemmas'] sentence_features['pos_tags'] += word_features['pos_tags'] sentence_features['feats'] += word_features['feats'] return sentence_features def build_token_list(sentence_features): return [ (idx, dediac_ar(token), lemma, pos_tag, '_', feats, '_', '_', '_', '_') for idx, (token, lemma, pos_tag, feats) in enumerate( zip(sentence_features['tokens'], sentence_features['lemmas'], sentence_features['pos_tags'], sentence_features['feats']) , 1) ] def to_conll_fields_list(sentence_analysis_list: List[List[dict]], clitic_feats, tagset): sentence_features_list = [] for sentence_analysis in sentence_analysis_list: sentence_features = {'tokens': [], 'lemmas': [], 'pos_tags': [], 'feats': []} for word_analysis in sentence_analysis: word_features_df = get_word_features_df(word_analysis, clitic_feats) word_features = join_feats(word_features_df, tagset) sentence_features = update_sentence_features(sentence_features, word_features) token_list = build_token_list(sentence_features) sentence_features_list.append(token_list) return sentence_features_list