Instructions to use WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 1,902 Bytes
356943f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | import torch
from fla.models.rwkv7 import (
RWKV7Config,
RWKV7ForCausalLM,
)
config = RWKV7Config(
attn_mode="chunk",
hidden_size=32,
hidden_ratio=4,
intermediate_size=32,
num_hidden_layers=2,
head_dim=32, #64,
num_heads=16,
decay_low_rank_dim=32,
gate_low_rank_dim=32,
a_low_rank_dim=32,
v_low_rank_dim=32,
hidden_act="sqrelu",
max_position_embeddings=2048,
norm_first=True,
norm_bias=True,
norm_eps=1e-5,
attn=None,
use_cache=False,
pad_token_id=0,
bos_token_id=2,
eos_token_id=3,
tie_word_embeddings=False,
initializer_range=0.02,
fuse_norm=False,
fuse_cross_entropy=True,
fuse_linear_cross_entropyl=False,
use_l2warp=False,
vocab_size=512,
value_dim=None,
)
device = "cuda" if torch.cuda.is_available() else "cpu"
model = RWKV7ForCausalLM(config).to(device)
print(
f"params: "
f"{sum(p.numel() for p in model.parameters()) / 1e6:.2f}M"
)
input_ids = torch.randint(
0,
config.vocab_size,
(2, 10),
device=device,
)
with torch.no_grad():
outputs = model(input_ids)
print(model.state_dict().keys())
print(model.config)
# 3. モデルのサイズ(パラメーター数)を確認してみる
total_params = sum(p.num_embeddings() if hasattr(p, "num_embeddings") else p.numel() for p in model.parameters())
print(f"設定したモデルの総パラメーター数: {total_params / 1e6:.2f} M (百万)")
print("出力テンソルの形状 (Logits):", outputs.logits.shape)
# 形状が [2, 10, 151936] (バッチ, トークン数, 語彙数) になっていれば大成功です!
import os
from safetensors.torch import save_file
output_dir = "hourai3_2"
os.makedirs(output_dir, exist_ok=True)
save_file(
model.state_dict(),
os.path.join(output_dir, "model.safetensors")
)
config.save_pretrained(output_dir)
|