Spaces:
Sleeping
Sleeping
File size: 3,545 Bytes
edad764 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | import sys
from typing import List
from camel_tools.disambig.common import DisambiguatedWord
from camel_tools.utils.dediac import dediac_ar
from ..classes import Token
def get_tok_match_analysis(disambig_word, token):
for i, scored_analysis in enumerate(disambig_word.analyses):
if dediac_ar(scored_analysis.analysis['diac']) == dediac_ar(token):
if i != 0:
# print(f"#NOT_FIRST Input token: {token}, analysis: {get_first_analysis(disambig_word)['diac']}", file=sys.stderr)
print(f"#NOT_FIRST {token}", file=sys.stderr)
return scored_analysis.analysis
# print(f"#NOAN token {token}, analysis: {get_first_analysis(disambig_word)['diac']}", file=sys.stderr)
print(f"#NOAN {token}", file=sys.stderr)
return get_first_analysis(disambig_word) # no token match, so just return first
def get_analysis_by_criteria(disambig_word: DisambiguatedWord, selection_criteria: dict) -> dict:
# runs assertions to ensure the data is good
is_analysis(disambig_word)
for scored_analysis in disambig_word.analyses:
analysis = scored_analysis.analysis
if all(analysis[key] == selection_criteria[key] for key in selection_criteria):
return analysis
# if the criteria is not found, return the first analysis
print(f"No analysis found for the given criteria on {disambig_word.word}, will return the first analysis.")
return get_first_analysis(disambig_word)
def is_analysis(disambig_word: DisambiguatedWord) -> dict:
# this should not be empty
assert disambig_word, "Disambiguated word not found!"
# this could be empty if there is no backoff, and this code assumes backoff
assert disambig_word.analyses, "No analyses found, are you using backoff?"
assert disambig_word.analyses[0], "No analyses found, are you using backoff?"
def get_first_analysis(disambig_word: DisambiguatedWord):
# runs assertions to ensure the data is good
is_analysis(disambig_word)
return disambig_word.analyses[0].analysis
def get_sentence_analysis(disambiguated_sentence: List[DisambiguatedWord], token_line, selection, selection_criteria) -> List[dict]:
# sourcery skip: switch
# get an analysis based on the selection criteria
# top means take first
# tok_match takes the first analysis that matches diac, otherwise takes first
# match means take the first analysis that matches the selection criteria.
# selection criteria: {'criteria1': 'value1', 'criteria2': 'value2'...}
# return None if criteria not fulfilled.
# TODO: check if selection criteria contains valid keys
if selection == 'top':
return [get_first_analysis(disambig_word) for disambig_word in disambiguated_sentence]
elif selection == 'tok_match':
return [get_tok_match_analysis(disambig_word, token) for disambig_word, token in zip(disambiguated_sentence, token_line)]
elif selection == 'match':
return [get_analysis_by_criteria(disambig_word, selection_criteria) for disambig_word in disambiguated_sentence]
else:
raise ValueError(f"the selection {selection} is not valid!")
def to_sentence_analysis_list(disambiguated_sentences: List[List[DisambiguatedWord]], token_lines, selection: str='tok_match', selection_criteria: dict=None) -> List[List[Token]]:
return [get_sentence_analysis(disambiguated_sentence, token_line, selection, selection_criteria) for disambiguated_sentence, token_line in zip(disambiguated_sentences, token_lines)]
|