Transformers
Safetensors
Japanese
rwkv7
text-embed
WariHima's picture
Upload 60 files
356943f verified
Raw History Blame Contribute Delete
1.9 kB
import torch
from fla.models.rwkv7 import (
RWKV7Config,
RWKV7ForCausalLM,
)
config = RWKV7Config(
attn_mode="chunk",
hidden_size=16,
hidden_ratio=4,
intermediate_size=32,
num_hidden_layers=2,
head_dim=16, #64,
num_heads=16,
decay_low_rank_dim=64,
gate_low_rank_dim=128,
a_low_rank_dim=64,
v_low_rank_dim=32,
hidden_act="sqrelu",
max_position_embeddings=2048,
norm_first=True,
norm_bias=True,
norm_eps=1e-5,
attn=None,
use_cache=False,
pad_token_id=0,
bos_token_id=2,
eos_token_id=3,
tie_word_embeddings=False,
initializer_range=0.02,
fuse_norm=False,
fuse_cross_entropy=True,
fuse_linear_cross_entropyl=False,
use_l2warp=False,
vocab_size=512,
value_dim=None,
)
device = "cuda" if torch.cuda.is_available() else "cpu"
model = RWKV7ForCausalLM(config).to(device)
print(
f"params: "
f"{sum(p.numel() for p in model.parameters()) / 1e6:.2f}M"
)
input_ids = torch.randint(
0,
config.vocab_size,
(2, 10),
device=device,
)
with torch.no_grad():
outputs = model(input_ids)
print(model.state_dict().keys())
print(model.config)
# 3. モデルのサイズ(パラメーター数)を確認してみる
total_params = sum(p.num_embeddings() if hasattr(p, "num_embeddings") else p.numel() for p in model.parameters())
print(f"設定したモデルの総パラメーター数: {total_params / 1e6:.2f} M (百万)")
print("出力テンソルの形状 (Logits):", outputs.logits.shape)
# 形状が [2, 10, 151936] (バッチ, トークン数, 語彙数) になっていれば大成功です!
import os
from safetensors.torch import save_file
output_dir = "hourai3_2"
os.makedirs(output_dir, exist_ok=True)
save_file(
model.state_dict(),
os.path.join(output_dir, "model.safetensors")
)
config.save_pretrained(output_dir)