Instructions to use WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download scripts/train_embed_c1_allrows_with_negatives.py from WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel: direct link, hf CLI and curl.
- Browser
- Download file 9.37 kB
-
https://huggingface.co/WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel/resolve/main/scripts/train_embed_c1_allrows_with_negatives.py
- Command line
-
hf download hf://WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel/scripts/train_embed_c1_allrows_with_negatives.py
-
curl -L -o train_embed_c1_allrows_with_negatives.py https://huggingface.co/WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel/resolve/main/scripts/train_embed_c1_allrows_with_negatives.py
9.37 kB
| from datasets import load_dataset, concatenate_datasets | |
| from dataclasses import dataclass, field | |
| import torch | |
| from sentence_transformers import ( | |
| SentenceTransformer, | |
| SentenceTransformerTrainer, | |
| SentenceTransformerTrainingArguments, | |
| ) | |
| from sentence_transformers.models import Transformer, Pooling | |
| from sentence_transformers.losses import MultipleNegativesRankingLoss | |
| from tokenization_japanese_hybrid import ( | |
| JapaneseHybridTokenizer, | |
| hybrid_encode, | |
| ) | |
| # ============================================================ | |
| # Settings | |
| # ============================================================ | |
| model_path = "./hourai3_2/" | |
| output_path = "./hourai3-allrows-negatives-embed_c1" | |
| MAX_SEQ_LENGTH = 512 | |
| TOKENIZE_BATCH_SIZE = 1000 | |
| TEXT_COLUMNS = ["anchor", "positive"] + [f"negative_{i}" for i in range(1, 9)] | |
| nli_files = [ | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__nu-snli-triplet/train-00000-of-00001.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jsnli-triplet/train-00000-of-00001.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__auto-wiki-nli-triplet/train-00000-of-00001.parquet", | |
| ] | |
| qa_files = [ | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00000-of-00007.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00001-of-00007.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00002-of-00007.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00003-of-00007.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00004-of-00007.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00005-of-00007.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jsquad/train-00006-of-00007.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00000-of-00005.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00001-of-00005.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00002-of-00005.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00003-of-00005.parquet", | |
| "./dataset/sentence_transformer_japanese/hpprc_emb__jaquad/train-00004-of-00005.parquet", | |
| ] | |
| # ============================================================ | |
| # Tokenizer | |
| # ============================================================ | |
| print("Loading JapaneseHybridTokenizer...") | |
| tokenizer = JapaneseHybridTokenizer.from_pretrained(model_path) | |
| pad_token_id = tokenizer._bpe.token_to_id(tokenizer.pad_token) | |
| if pad_token_id is None: | |
| raise ValueError( | |
| f"{tokenizer.pad_token!r} is missing from tokenizer.json" | |
| ) | |
| if tokenizer.pad_token_id != pad_token_id: | |
| raise ValueError( | |
| f"Tokenizer pad ID mismatch: wrapper={tokenizer.pad_token_id}, " | |
| f"Rust backend={pad_token_id}" | |
| ) | |
| print( | |
| f"Tokenizer ready: vocab={tokenizer.vocab_size}, " | |
| f"pad_token_id={pad_token_id}" | |
| ) | |
| # ============================================================ | |
| # Load and combine datasets | |
| # ============================================================ | |
| print("Loading NLI datasets...") | |
| nli_dataset = load_dataset("parquet", data_files=nli_files, split="train") | |
| nli_dataset = nli_dataset.shuffle(seed=42) | |
| train_nli_dataset = nli_dataset | |
| print(f"NLI rows: {len(train_nli_dataset):,}") | |
| print("Loading QA datasets...") | |
| qa_dataset = load_dataset("parquet", data_files=qa_files, split="train") | |
| qa_dataset = qa_dataset.shuffle(seed=42) | |
| train_qa_dataset = qa_dataset | |
| print(f"QA rows: {len(train_qa_dataset):,}") | |
| # Use every row from all listed NLI and QA Parquet files. | |
| train_dataset = concatenate_datasets( | |
| [train_nli_dataset, train_qa_dataset] | |
| ).shuffle(seed=42) | |
| print(f"Total training rows: {len(train_dataset):,}") | |
| # Use anchor/positive plus all eight explicit hard-negative columns. | |
| # MultipleNegativesRankingLoss treats columns after positive as hard negatives. | |
| missing_columns = [ | |
| column for column in TEXT_COLUMNS | |
| if column not in train_dataset.column_names | |
| ] | |
| if missing_columns: | |
| raise ValueError( | |
| "Dataset is missing required training columns: " | |
| f"{missing_columns}. Available columns: {train_dataset.column_names}" | |
| ) | |
| train_dataset = train_dataset.select_columns(TEXT_COLUMNS) | |
| # ============================================================ | |
| # Rust batch tokenization (offline preprocessing) | |
| # ============================================================ | |
| def tokenize_dataset(batch): | |
| result = {} | |
| for column in TEXT_COLUMNS: | |
| texts = [ | |
| hybrid_encode(text if text is not None else "") | |
| for text in batch[column] | |
| ] | |
| encodings = tokenizer._bpe.encode_batch( | |
| texts, | |
| add_special_tokens=False, | |
| ) | |
| result[column] = [ | |
| encoding.ids[:MAX_SEQ_LENGTH] | |
| for encoding in encodings | |
| ] | |
| return result | |
| print("Tokenizing train dataset...") | |
| train_dataset = train_dataset.map( | |
| tokenize_dataset, | |
| batched=True, | |
| batch_size=TOKENIZE_BATCH_SIZE, | |
| remove_columns=TEXT_COLUMNS, | |
| desc="Tokenizing train dataset", | |
| ) | |
| print("Tokenized columns:", train_dataset.column_names) | |
| print( | |
| "Tokenization example:", | |
| {column: train_dataset[0][column][:32] for column in TEXT_COLUMNS}, | |
| ) | |
| print("Tokenized rows:", len(train_dataset)) | |
| # ============================================================ | |
| # Custom collator for pre-tokenized IDs | |
| # | |
| # SentenceTransformer's default collator expects strings and calls | |
| # model.preprocess(). This collator instead pads already-tokenized IDs | |
| # and returns the exact *_input_ids / *_attention_mask keys consumed | |
| # by SentenceTransformerTrainer.collect_features(). | |
| # ============================================================ | |
| class PreTokenizedCollator: | |
| pad_token_id: int | |
| max_length: int = 512 | |
| # Required by SentenceTransformerTrainer when constructing its sampler. | |
| # These are label-column names, not text columns. | |
| valid_label_columns: list[str] = field( | |
| default_factory=lambda: ["label", "labels", "score", "scores"] | |
| ) | |
| def __call__(self, features): | |
| batch = {} | |
| for column in TEXT_COLUMNS: | |
| sequences = [ | |
| row[column][: self.max_length] | |
| for row in features | |
| ] | |
| batch_max_length = max( | |
| 1, | |
| max(len(sequence) for sequence in sequences), | |
| ) | |
| input_ids = torch.full( | |
| (len(sequences), batch_max_length), | |
| fill_value=self.pad_token_id, | |
| dtype=torch.long, | |
| ) | |
| attention_mask = torch.zeros( | |
| (len(sequences), batch_max_length), | |
| dtype=torch.long, | |
| ) | |
| for index, sequence in enumerate(sequences): | |
| if not sequence: | |
| continue | |
| length = len(sequence) | |
| input_ids[index, :length] = torch.tensor( | |
| sequence, | |
| dtype=torch.long, | |
| ) | |
| attention_mask[index, :length] = 1 | |
| batch[f"{column}_input_ids"] = input_ids | |
| batch[f"{column}_attention_mask"] = attention_mask | |
| return batch | |
| # ============================================================ | |
| # SentenceTransformer model | |
| # ============================================================ | |
| word_embedding_model = Transformer( | |
| model_name_or_path=model_path, | |
| max_seq_length=MAX_SEQ_LENGTH, | |
| model_kwargs={ | |
| "trust_remote_code": True, | |
| }, | |
| ) | |
| model_tokenizer_pad_id = word_embedding_model.tokenizer.pad_token_id | |
| if model_tokenizer_pad_id != pad_token_id: | |
| raise ValueError( | |
| "Model tokenizer and hybrid tokenizer pad IDs differ: " | |
| f"model={model_tokenizer_pad_id}, hybrid={pad_token_id}" | |
| ) | |
| pooling_model = Pooling( | |
| word_embedding_dimension=word_embedding_model.get_word_embedding_dimension(), | |
| pooling_mode="lasttoken", | |
| ) | |
| model = SentenceTransformer( | |
| modules=[word_embedding_model, pooling_model] | |
| ) | |
| loss_fn = MultipleNegativesRankingLoss(model=model) | |
| # ============================================================ | |
| # Training | |
| # ============================================================ | |
| training_args = SentenceTransformerTrainingArguments( | |
| output_dir="./outputs-allrows-negatives-embed_c1", | |
| num_train_epochs=1, | |
| per_device_train_batch_size=100, | |
| learning_rate=1e-4, | |
| warmup_steps=300, | |
| max_grad_norm=1.0, | |
| bf16=True, | |
| logging_steps=5, | |
| report_to="none", | |
| gradient_checkpointing=True, | |
| gradient_checkpointing_kwargs={"use_reentrant": False}, | |
| dataloader_num_workers=4, | |
| ) | |
| trainer = SentenceTransformerTrainer( | |
| model=model, | |
| args=training_args, | |
| train_dataset=train_dataset, | |
| loss=loss_fn, | |
| data_collator=PreTokenizedCollator( | |
| pad_token_id=pad_token_id, | |
| max_length=MAX_SEQ_LENGTH, | |
| ), | |
| ) | |
| print("Trainer example:", trainer.train_dataset[0]) | |
| print("Starting Fine-Tuning...") | |
| trainer.train(resume_from_checkpoint=False) | |
| # ============================================================ | |
| # Save | |
| # ============================================================ | |
| model.save_pretrained(output_path) | |
| print(f"Saved successfully: {output_path}") | |