Spaces:
Runtime error
Runtime error
File size: 4,444 Bytes
fc7db98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | """Shared constants and helpers for the word complexity project."""
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent
DATA_DIR = PROJECT_ROOT / "data"
OUTPUT_DIR = PROJECT_ROOT / "outputs"
CHECKPOINT_DIR = PROJECT_ROOT / "checkpoints"
EXPORT_DIR = PROJECT_ROOT / "exported_models"
LEVEL_ORDER = ["Very Easy", "Easy", "Medium", "Hard", "Very Hard"]
REASON_ORDER = ["Lexical Rarity", "Contextual Ambiguity", "Syntactic Complexity"]
HARD_LEVELS = {"Hard", "Very Hard"}
LEVEL_TO_ID = {level: idx for idx, level in enumerate(LEVEL_ORDER)}
ID_TO_LEVEL = {idx: level for level, idx in LEVEL_TO_ID.items()}
REASON_TO_ID = {reason: idx for idx, reason in enumerate(REASON_ORDER)}
ID_TO_REASON = {idx: reason for reason, idx in REASON_TO_ID.items()}
NONE_REASON_ID = -1
TGT_TOKEN = "[TGT]"
TGT_END_TOKEN = "[/TGT]"
# Input encoding strategies (SemEval / ABSA literature)
ENCODING_SPAN_MARK = "span_mark" # [TGT] word [/TGT] inside sentence
ENCODING_PAIR_CAMBRIDGE = "pair_cambridge" # [CLS] target [SEP] sentence
ENCODING_PAIR_CONTEXT = "pair_context" # sentence [SEP] target (SemEval common)
# Pooling strategies for target-focused readout
POOLING_SPAN = "span" # mean of tokens inside marked span (default)
POOLING_CLS_CONCAT = "cls_concat" # legacy: [CLS] + first [TGT]
POOLING_TGT_MARKER = "tgt_marker" # first [TGT] token only
POOLING_CLS_ONLY = "cls_only" # [CLS] only (for pair encoding)
MODELS = {
"deberta": "microsoft/deberta-v3-base",
"distilbert": "distilbert-base-uncased",
"roberta": "roberta-base",
}
MERGE_KEYS = ["id", "sentence", "target_word", "complexity_level"]
def level_id(level: str) -> int:
return LEVEL_TO_ID[level]
def reason_id(reason: str) -> int:
if reason in ("NONE", None) or (isinstance(reason, float) and str(reason) == "nan"):
return NONE_REASON_ID
return REASON_TO_ID[reason]
def is_hard_level(level: str) -> bool:
return level in HARD_LEVELS
def wrap_target_word(sentence: str, target_word: str) -> str:
"""Wrap target word with open/close span markers (ABSA aspect-marker style)."""
if not target_word or target_word not in sentence:
return sentence
marked = f"{TGT_TOKEN} {target_word} {TGT_END_TOKEN}"
return sentence.replace(target_word, marked, 1)
def build_model_input(
sentence: str,
target_word: str,
encoding: str = ENCODING_SPAN_MARK,
corpus: str | None = None,
) -> str | tuple[str, str]:
"""
Build tokenizer input for LCP.
Returns a string for span marking, or (text_a, text_b) for pair encodings.
"""
sentence = str(sentence)
target_word = str(target_word)
if encoding == ENCODING_SPAN_MARK:
return wrap_target_word(sentence, target_word)
if encoding == ENCODING_PAIR_CAMBRIDGE:
return target_word, sentence
if encoding == ENCODING_PAIR_CONTEXT:
if corpus:
return f"{corpus} {target_word}", sentence
return sentence, target_word
raise ValueError(f"Unknown encoding: {encoding}")
def difficult_class_probability(level_probs) -> float:
"""P(Hard) + P(Very Hard) from the 5-class softmax — auxiliary only, not a regression score."""
if hasattr(level_probs, "tolist"):
level_probs = level_probs.tolist()
if isinstance(level_probs, dict):
return float(level_probs.get("Hard", 0) + level_probs.get("Very Hard", 0))
return float(level_probs[3] + level_probs[4])
def ensure_dirs() -> None:
DATA_DIR.mkdir(parents=True, exist_ok=True)
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
EXPORT_DIR.mkdir(parents=True, exist_ok=True)
def tokenize_lcp_input(
tokenizer,
sentence: str,
target_word: str,
encoding: str = ENCODING_SPAN_MARK,
max_length: int = 192,
corpus: str | None = None,
):
"""Tokenize a (sentence, target_word) pair for the LCP model."""
built = build_model_input(sentence, target_word, encoding=encoding, corpus=corpus)
if isinstance(built, tuple):
return tokenizer(
built[0],
built[1],
truncation=True,
max_length=max_length,
padding="max_length",
return_tensors="pt",
)
return tokenizer(
built,
truncation=True,
max_length=max_length,
padding="max_length",
return_tensors="pt",
)
|