import torch from fla.models.rwkv7 import ( RWKV7Config, RWKV7ForCausalLM, ) config = RWKV7Config( attn_mode="chunk", hidden_size=32, hidden_ratio=4, intermediate_size=32, num_hidden_layers=2, head_dim=32, #64, num_heads=16, decay_low_rank_dim=32, gate_low_rank_dim=32, a_low_rank_dim=32, v_low_rank_dim=32, hidden_act="sqrelu", max_position_embeddings=2048, norm_first=True, norm_bias=True, norm_eps=1e-5, attn=None, use_cache=False, pad_token_id=0, bos_token_id=2, eos_token_id=3, tie_word_embeddings=False, initializer_range=0.02, fuse_norm=False, fuse_cross_entropy=True, fuse_linear_cross_entropyl=False, use_l2warp=False, vocab_size=512, value_dim=None, ) device = "cuda" if torch.cuda.is_available() else "cpu" model = RWKV7ForCausalLM(config).to(device) print( f"params: " f"{sum(p.numel() for p in model.parameters()) / 1e6:.2f}M" ) input_ids = torch.randint( 0, config.vocab_size, (2, 10), device=device, ) with torch.no_grad(): outputs = model(input_ids) print(model.state_dict().keys()) print(model.config) # 3. モデルのサイズ(パラメーター数)を確認してみる total_params = sum(p.num_embeddings() if hasattr(p, "num_embeddings") else p.numel() for p in model.parameters()) print(f"設定したモデルの総パラメーター数: {total_params / 1e6:.2f} M (百万)") print("出力テンソルの形状 (Logits):", outputs.logits.shape) # 形状が [2, 10, 151936] (バッチ, トークン数, 語彙数) になっていれば大成功です! import os from safetensors.torch import save_file output_dir = "hourai3_2" os.makedirs(output_dir, exist_ok=True) save_file( model.state_dict(), os.path.join(output_dir, "model.safetensors") ) config.save_pretrained(output_dir)