Transformers
Safetensors
Japanese
rwkv7
text-embed
File size: 1,902 Bytes
356943f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
import torch

from fla.models.rwkv7 import (
    RWKV7Config,
    RWKV7ForCausalLM,
)


config = RWKV7Config(
    attn_mode="chunk",

    hidden_size=16,
    hidden_ratio=4,
    intermediate_size=32,

    num_hidden_layers=2,
    head_dim=16, #64,
    num_heads=16,

    decay_low_rank_dim=64,
    gate_low_rank_dim=128,
    a_low_rank_dim=64,
    v_low_rank_dim=32,

    hidden_act="sqrelu",

    max_position_embeddings=2048,

    norm_first=True,
    norm_bias=True,
    norm_eps=1e-5,

    attn=None,
    use_cache=False,

    pad_token_id=0,
    bos_token_id=2,
    eos_token_id=3,

    tie_word_embeddings=False,

    initializer_range=0.02,

    fuse_norm=False,
    fuse_cross_entropy=True,
    fuse_linear_cross_entropyl=False,
    use_l2warp=False,

    vocab_size=512,
    value_dim=None,
)

device = "cuda" if torch.cuda.is_available() else "cpu"

model = RWKV7ForCausalLM(config).to(device)

print(
    f"params: "
    f"{sum(p.numel() for p in model.parameters()) / 1e6:.2f}M"
)

input_ids = torch.randint(
    0,
    config.vocab_size,
    (2, 10),
    device=device,
)

with torch.no_grad():
    outputs = model(input_ids)

print(model.state_dict().keys())
print(model.config)

# 3. モデルのサイズ(パラメーター数)を確認してみる
total_params = sum(p.num_embeddings() if hasattr(p, "num_embeddings") else p.numel() for p in model.parameters())
print(f"設定したモデルの総パラメーター数: {total_params / 1e6:.2f} M (百万)")

print("出力テンソルの形状 (Logits):", outputs.logits.shape) 
# 形状が [2, 10, 151936] (バッチ, トークン数, 語彙数) になっていれば大成功です!



import os
from safetensors.torch import save_file

output_dir = "hourai3_2"
os.makedirs(output_dir, exist_ok=True)

save_file(
    model.state_dict(),
    os.path.join(output_dir, "model.safetensors")
)

config.save_pretrained(output_dir)