from datasets import load_dataset, concatenate_datasets from dataclasses import dataclass, field import torch from sentence_transformers import ( SentenceTransformer, SentenceTransformerTrainer, SentenceTransformerTrainingArguments, ) from sentence_transformers.models import Transformer, Pooling from sentence_transformers.losses import MultipleNegativesRankingLoss from tokenization_japanese_hybrid import ( JapaneseHybridTokenizer, hybrid_encode, ) # ============================================================ # Settings # ============================================================ model_path = "./hourai3_2/" output_path = "./hourai3-allrows-negatives-embed_c1" MAX_SEQ_LENGTH = 512 TOKENIZE_BATCH_SIZE = 1000 TEXT_COLUMNS = ["anchor", "positive"] + [f"negative_{i}" for i in range(1, 9)] nli_files = [ "./dataset/sentence_transformer_japanese/hpprc_emb__nu-snli-triplet/train-00000-of-00001.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jsnli-triplet/train-00000-of-00001.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__auto-wiki-nli-triplet/train-00000-of-00001.parquet", ] qa_files = [ "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00000-of-00007.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00001-of-00007.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00002-of-00007.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00003-of-00007.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00004-of-00007.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00005-of-00007.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00006-of-00007.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00000-of-00005.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00001-of-00005.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00002-of-00005.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00003-of-00005.parquet", "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00004-of-00005.parquet", ] # ============================================================ # Tokenizer # ============================================================ print("Loading JapaneseHybridTokenizer...") tokenizer = JapaneseHybridTokenizer.from_pretrained(model_path) pad_token_id = tokenizer._bpe.token_to_id(tokenizer.pad_token) if pad_token_id is None: raise ValueError( f"{tokenizer.pad_token!r} is missing from tokenizer.json" ) if tokenizer.pad_token_id != pad_token_id: raise ValueError( f"Tokenizer pad ID mismatch: wrapper={tokenizer.pad_token_id}, " f"Rust backend={pad_token_id}" ) print( f"Tokenizer ready: vocab={tokenizer.vocab_size}, " f"pad_token_id={pad_token_id}" ) # ============================================================ # Load and combine datasets # ============================================================ print("Loading NLI datasets...") nli_dataset = load_dataset("parquet", data_files=nli_files, split="train") nli_dataset = nli_dataset.shuffle(seed=42) train_nli_dataset = nli_dataset print(f"NLI rows: {len(train_nli_dataset):,}") print("Loading QA datasets...") qa_dataset = load_dataset("parquet", data_files=qa_files, split="train") qa_dataset = qa_dataset.shuffle(seed=42) train_qa_dataset = qa_dataset print(f"QA rows: {len(train_qa_dataset):,}") # Use every row from all listed NLI and QA Parquet files. train_dataset = concatenate_datasets( [train_nli_dataset, train_qa_dataset] ).shuffle(seed=42) print(f"Total training rows: {len(train_dataset):,}") # Use anchor/positive plus all eight explicit hard-negative columns. # MultipleNegativesRankingLoss treats columns after positive as hard negatives. missing_columns = [ column for column in TEXT_COLUMNS if column not in train_dataset.column_names ] if missing_columns: raise ValueError( "Dataset is missing required training columns: " f"{missing_columns}. Available columns: {train_dataset.column_names}" ) train_dataset = train_dataset.select_columns(TEXT_COLUMNS) # ============================================================ # Rust batch tokenization (offline preprocessing) # ============================================================ def tokenize_dataset(batch): result = {} for column in TEXT_COLUMNS: texts = [ hybrid_encode(text if text is not None else "") for text in batch[column] ] encodings = tokenizer._bpe.encode_batch( texts, add_special_tokens=False, ) result[column] = [ encoding.ids[:MAX_SEQ_LENGTH] for encoding in encodings ] return result print("Tokenizing train dataset...") train_dataset = train_dataset.map( tokenize_dataset, batched=True, batch_size=TOKENIZE_BATCH_SIZE, remove_columns=TEXT_COLUMNS, desc="Tokenizing train dataset", ) print("Tokenized columns:", train_dataset.column_names) print( "Tokenization example:", {column: train_dataset[0][column][:32] for column in TEXT_COLUMNS}, ) print("Tokenized rows:", len(train_dataset)) # ============================================================ # Custom collator for pre-tokenized IDs # # SentenceTransformer's default collator expects strings and calls # model.preprocess(). This collator instead pads already-tokenized IDs # and returns the exact *_input_ids / *_attention_mask keys consumed # by SentenceTransformerTrainer.collect_features(). # ============================================================ @dataclass class PreTokenizedCollator: pad_token_id: int max_length: int = 512 # Required by SentenceTransformerTrainer when constructing its sampler. # These are label-column names, not text columns. valid_label_columns: list[str] = field( default_factory=lambda: ["label", "labels", "score", "scores"] ) def __call__(self, features): batch = {} for column in TEXT_COLUMNS: sequences = [ row[column][: self.max_length] for row in features ] batch_max_length = max( 1, max(len(sequence) for sequence in sequences), ) input_ids = torch.full( (len(sequences), batch_max_length), fill_value=self.pad_token_id, dtype=torch.long, ) attention_mask = torch.zeros( (len(sequences), batch_max_length), dtype=torch.long, ) for index, sequence in enumerate(sequences): if not sequence: continue length = len(sequence) input_ids[index, :length] = torch.tensor( sequence, dtype=torch.long, ) attention_mask[index, :length] = 1 batch[f"{column}_input_ids"] = input_ids batch[f"{column}_attention_mask"] = attention_mask return batch # ============================================================ # SentenceTransformer model # ============================================================ word_embedding_model = Transformer( model_name_or_path=model_path, max_seq_length=MAX_SEQ_LENGTH, model_kwargs={ "trust_remote_code": True, }, ) model_tokenizer_pad_id = word_embedding_model.tokenizer.pad_token_id if model_tokenizer_pad_id != pad_token_id: raise ValueError( "Model tokenizer and hybrid tokenizer pad IDs differ: " f"model={model_tokenizer_pad_id}, hybrid={pad_token_id}" ) pooling_model = Pooling( word_embedding_dimension=word_embedding_model.get_word_embedding_dimension(), pooling_mode="lasttoken", ) model = SentenceTransformer( modules=[word_embedding_model, pooling_model] ) loss_fn = MultipleNegativesRankingLoss(model=model) # ============================================================ # Training # ============================================================ training_args = SentenceTransformerTrainingArguments( output_dir="./outputs-allrows-negatives-embed_c1", num_train_epochs=1, per_device_train_batch_size=100, learning_rate=1e-4, warmup_steps=300, max_grad_norm=1.0, bf16=True, logging_steps=5, report_to="none", gradient_checkpointing=True, gradient_checkpointing_kwargs={"use_reentrant": False}, dataloader_num_workers=4, ) trainer = SentenceTransformerTrainer( model=model, args=training_args, train_dataset=train_dataset, loss=loss_fn, data_collator=PreTokenizedCollator( pad_token_id=pad_token_id, max_length=MAX_SEQ_LENGTH, ), ) print("Trainer example:", trainer.train_dataset[0]) print("Starting Fine-Tuning...") trainer.train(resume_from_checkpoint=False) # ============================================================ # Save # ============================================================ model.save_pretrained(output_path) print(f"Saved successfully: {output_path}")