Transformers
Safetensors
Japanese
rwkv7
text-embed
DanMatsu-KilosizeJapaneseTextEmbedModel / scripts /train_embed_c1_allrows_with_negatives.py
WariHima's picture
Upload 60 files
356943f verified
Raw History Blame Contribute Delete
9.37 kB
from datasets import load_dataset, concatenate_datasets
from dataclasses import dataclass, field
import torch
from sentence_transformers import (
SentenceTransformer,
SentenceTransformerTrainer,
SentenceTransformerTrainingArguments,
)
from sentence_transformers.models import Transformer, Pooling
from sentence_transformers.losses import MultipleNegativesRankingLoss
from tokenization_japanese_hybrid import (
JapaneseHybridTokenizer,
hybrid_encode,
)
# ============================================================
# Settings
# ============================================================
model_path = "./hourai3_2/"
output_path = "./hourai3-allrows-negatives-embed_c1"
MAX_SEQ_LENGTH = 512
TOKENIZE_BATCH_SIZE = 1000
TEXT_COLUMNS = ["anchor", "positive"] + [f"negative_{i}" for i in range(1, 9)]
nli_files = [
"./dataset/sentence_transformer_japanese/hpprc_emb__nu-snli-triplet/train-00000-of-00001.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jsnli-triplet/train-00000-of-00001.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__auto-wiki-nli-triplet/train-00000-of-00001.parquet",
]
qa_files = [
"./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00000-of-00007.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00001-of-00007.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00002-of-00007.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00003-of-00007.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00004-of-00007.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00005-of-00007.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00006-of-00007.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00000-of-00005.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00001-of-00005.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00002-of-00005.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00003-of-00005.parquet",
"./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00004-of-00005.parquet",
]
# ============================================================
# Tokenizer
# ============================================================
print("Loading JapaneseHybridTokenizer...")
tokenizer = JapaneseHybridTokenizer.from_pretrained(model_path)
pad_token_id = tokenizer._bpe.token_to_id(tokenizer.pad_token)
if pad_token_id is None:
raise ValueError(
f"{tokenizer.pad_token!r} is missing from tokenizer.json"
)
if tokenizer.pad_token_id != pad_token_id:
raise ValueError(
f"Tokenizer pad ID mismatch: wrapper={tokenizer.pad_token_id}, "
f"Rust backend={pad_token_id}"
)
print(
f"Tokenizer ready: vocab={tokenizer.vocab_size}, "
f"pad_token_id={pad_token_id}"
)
# ============================================================
# Load and combine datasets
# ============================================================
print("Loading NLI datasets...")
nli_dataset = load_dataset("parquet", data_files=nli_files, split="train")
nli_dataset = nli_dataset.shuffle(seed=42)
train_nli_dataset = nli_dataset
print(f"NLI rows: {len(train_nli_dataset):,}")
print("Loading QA datasets...")
qa_dataset = load_dataset("parquet", data_files=qa_files, split="train")
qa_dataset = qa_dataset.shuffle(seed=42)
train_qa_dataset = qa_dataset
print(f"QA rows: {len(train_qa_dataset):,}")
# Use every row from all listed NLI and QA Parquet files.
train_dataset = concatenate_datasets(
[train_nli_dataset, train_qa_dataset]
).shuffle(seed=42)
print(f"Total training rows: {len(train_dataset):,}")
# Use anchor/positive plus all eight explicit hard-negative columns.
# MultipleNegativesRankingLoss treats columns after positive as hard negatives.
missing_columns = [
column for column in TEXT_COLUMNS
if column not in train_dataset.column_names
]
if missing_columns:
raise ValueError(
"Dataset is missing required training columns: "
f"{missing_columns}. Available columns: {train_dataset.column_names}"
)
train_dataset = train_dataset.select_columns(TEXT_COLUMNS)
# ============================================================
# Rust batch tokenization (offline preprocessing)
# ============================================================
def tokenize_dataset(batch):
result = {}
for column in TEXT_COLUMNS:
texts = [
hybrid_encode(text if text is not None else "")
for text in batch[column]
]
encodings = tokenizer._bpe.encode_batch(
texts,
add_special_tokens=False,
)
result[column] = [
encoding.ids[:MAX_SEQ_LENGTH]
for encoding in encodings
]
return result
print("Tokenizing train dataset...")
train_dataset = train_dataset.map(
tokenize_dataset,
batched=True,
batch_size=TOKENIZE_BATCH_SIZE,
remove_columns=TEXT_COLUMNS,
desc="Tokenizing train dataset",
)
print("Tokenized columns:", train_dataset.column_names)
print(
"Tokenization example:",
{column: train_dataset[0][column][:32] for column in TEXT_COLUMNS},
)
print("Tokenized rows:", len(train_dataset))
# ============================================================
# Custom collator for pre-tokenized IDs
#
# SentenceTransformer's default collator expects strings and calls
# model.preprocess(). This collator instead pads already-tokenized IDs
# and returns the exact *_input_ids / *_attention_mask keys consumed
# by SentenceTransformerTrainer.collect_features().
# ============================================================
@dataclass
class PreTokenizedCollator:
pad_token_id: int
max_length: int = 512
# Required by SentenceTransformerTrainer when constructing its sampler.
# These are label-column names, not text columns.
valid_label_columns: list[str] = field(
default_factory=lambda: ["label", "labels", "score", "scores"]
)
def __call__(self, features):
batch = {}
for column in TEXT_COLUMNS:
sequences = [
row[column][: self.max_length]
for row in features
]
batch_max_length = max(
1,
max(len(sequence) for sequence in sequences),
)
input_ids = torch.full(
(len(sequences), batch_max_length),
fill_value=self.pad_token_id,
dtype=torch.long,
)
attention_mask = torch.zeros(
(len(sequences), batch_max_length),
dtype=torch.long,
)
for index, sequence in enumerate(sequences):
if not sequence:
continue
length = len(sequence)
input_ids[index, :length] = torch.tensor(
sequence,
dtype=torch.long,
)
attention_mask[index, :length] = 1
batch[f"{column}_input_ids"] = input_ids
batch[f"{column}_attention_mask"] = attention_mask
return batch
# ============================================================
# SentenceTransformer model
# ============================================================
word_embedding_model = Transformer(
model_name_or_path=model_path,
max_seq_length=MAX_SEQ_LENGTH,
model_kwargs={
"trust_remote_code": True,
},
)
model_tokenizer_pad_id = word_embedding_model.tokenizer.pad_token_id
if model_tokenizer_pad_id != pad_token_id:
raise ValueError(
"Model tokenizer and hybrid tokenizer pad IDs differ: "
f"model={model_tokenizer_pad_id}, hybrid={pad_token_id}"
)
pooling_model = Pooling(
word_embedding_dimension=word_embedding_model.get_word_embedding_dimension(),
pooling_mode="lasttoken",
)
model = SentenceTransformer(
modules=[word_embedding_model, pooling_model]
)
loss_fn = MultipleNegativesRankingLoss(model=model)
# ============================================================
# Training
# ============================================================
training_args = SentenceTransformerTrainingArguments(
output_dir="./outputs-allrows-negatives-embed_c1",
num_train_epochs=1,
per_device_train_batch_size=100,
learning_rate=1e-4,
warmup_steps=300,
max_grad_norm=1.0,
bf16=True,
logging_steps=5,
report_to="none",
gradient_checkpointing=True,
gradient_checkpointing_kwargs={"use_reentrant": False},
dataloader_num_workers=4,
)
trainer = SentenceTransformerTrainer(
model=model,
args=training_args,
train_dataset=train_dataset,
loss=loss_fn,
data_collator=PreTokenizedCollator(
pad_token_id=pad_token_id,
max_length=MAX_SEQ_LENGTH,
),
)
print("Trainer example:", trainer.train_dataset[0])
print("Starting Fine-Tuning...")
trainer.train(resume_from_checkpoint=False)
# ============================================================
# Save
# ============================================================
model.save_pretrained(output_path)
print(f"Saved successfully: {output_path}")