File size: 3,048 Bytes
6dc7c27 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | import pytorch_lightning as pl
from tqdm import tqdm
from src.consec_dataset import build_samples_generator_from_disambiguation_corpus, ConsecDataset
from src.consec_tokenizer import ConsecTokenizer, DeBERTaTokenizer
from src.dependency_finder import EmptyDependencyFinder, PPMIPolysemyDependencyFinder
from src.disambiguation_corpora import WordNetCorpus
from src.sense_inventories import WordNetSenseInventory, XlWSDSenseInventory
if __name__ == "__main__":
pl.seed_everything(seed=96)
tokenizer = DeBERTaTokenizer(
transformer_model="microsoft/deberta-large",
# target_marker=("{{{", "}}}"),
target_marker=("<d>", "</d>"),
context_definitions_token="CONTEXT_DEFS",
context_markers=dict(number=1, pattern=("DEF_SEP", "DEF_END")),
# context_markers=dict(number=100, pattern=("<c#I#>", "</c#I#>")),
add_prefix_space=True,
)
wordnet = XlWSDSenseInventory("data/xl-wsd/inventories/inventory.it.txt", "data/xl-wsd/synset2definition.it.tsv")
disambiguation_corpus = WordNetCorpus(
# "data/WSD_Evaluation_Framework/Training_Corpora/SemCor/semcor",
"data/xl-wsd/evaluation_datasets/dev-it/dev-it",
# "data/WSD_Evaluation_Framework/Evaluation_Datasets/ALL/ALL",
materialize=False,
cached=False,
)
dependency_finder = PPMIPolysemyDependencyFinder(
sense_inventory=wordnet,
single_counter_path="data/pmi/it/words_counter.tsv",
pair_counter_path="data/pmi/it/word_pairs_counter.tsv",
energy=0.7,
minimum_ppmi=0.1,
max_dependencies=9,
with_pos=False,
)
generate_samples = build_samples_generator_from_disambiguation_corpus(
sense_inventory=wordnet,
disambiguation_corpus=disambiguation_corpus,
dependency_finder=dependency_finder,
sentence_window=2,
randomize_sentence_window=True,
remove_multilabel_instances=True,
shuffle_definitions=True,
randomize_dependencies=False,
)
consec_dataset = ConsecDataset(
samples_generator=generate_samples,
tokenizer=tokenizer,
use_definition_start=True,
# text_encoding_strategy="positional",
text_encoding_strategy="relative-positions",
tokens_per_batch=1536,
max_batch_size=128,
section_size=10,
prebatch=True,
shuffle=True,
max_length=tokenizer.model_max_length,
)
depends_from_counts = []
with open("/tmp/consec.log", "w") as f:
for dataset_sample in tqdm(consec_dataset.dataset_iterator_func()):
depends_from_counts.append(len(dataset_sample["context_definitions"]))
f.write(f'{tokenizer.tokenizer.decode(dataset_sample["input_ids"], skip_special_tokens=False)}\n')
print("Dependencies avg num: {}".format(sum(depends_from_counts) / len(depends_from_counts)))
print("Dependencies max num: {}".format(max(depends_from_counts)))
|