Instructions to use WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download scripts/create_embed20k.py from WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel: direct link, hf CLI and curl.
- Browser
- Download file 1.9 kB
-
https://huggingface.co/WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel/resolve/main/scripts/create_embed20k.py
- Command line
-
hf download hf://WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel/scripts/create_embed20k.py
-
curl -L -o create_embed20k.py https://huggingface.co/WariHima/DanMatsu-KilosizeJapaneseTextEmbedModel/resolve/main/scripts/create_embed20k.py
1.9 kB
| import torch | |
| from fla.models.rwkv7 import ( | |
| RWKV7Config, | |
| RWKV7ForCausalLM, | |
| ) | |
| config = RWKV7Config( | |
| attn_mode="chunk", | |
| hidden_size=16, | |
| hidden_ratio=4, | |
| intermediate_size=16, | |
| num_hidden_layers=2, | |
| head_dim=16, #64, | |
| num_heads=16, | |
| decay_low_rank_dim=16, | |
| gate_low_rank_dim=16, | |
| a_low_rank_dim=16, | |
| v_low_rank_dim=16, | |
| hidden_act="sqrelu", | |
| max_position_embeddings=2048, | |
| norm_first=True, | |
| norm_bias=True, | |
| norm_eps=1e-5, | |
| attn=None, | |
| use_cache=False, | |
| pad_token_id=0, | |
| bos_token_id=2, | |
| eos_token_id=3, | |
| tie_word_embeddings=False, | |
| initializer_range=0.02, | |
| fuse_norm=False, | |
| fuse_cross_entropy=True, | |
| fuse_linear_cross_entropyl=False, | |
| use_l2warp=False, | |
| vocab_size=512, | |
| value_dim=None, | |
| ) | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| model = RWKV7ForCausalLM(config).to(device) | |
| print( | |
| f"params: " | |
| f"{sum(p.numel() for p in model.parameters()) / 1e6:.2f}M" | |
| ) | |
| input_ids = torch.randint( | |
| 0, | |
| config.vocab_size, | |
| (2, 10), | |
| device=device, | |
| ) | |
| with torch.no_grad(): | |
| outputs = model(input_ids) | |
| print(model.state_dict().keys()) | |
| print(model.config) | |
| # 3. モデルのサイズ(パラメーター数)を確認してみる | |
| total_params = sum(p.num_embeddings() if hasattr(p, "num_embeddings") else p.numel() for p in model.parameters()) | |
| print(f"設定したモデルの総パラメーター数: {total_params / 1e6:.2f} M (百万)") | |
| print("出力テンソルの形状 (Logits):", outputs.logits.shape) | |
| # 形状が [2, 10, 151936] (バッチ, トークン数, 語彙数) になっていれば大成功です! | |
| import os | |
| from safetensors.torch import save_file | |
| output_dir = "hourai3_2" | |
| os.makedirs(output_dir, exist_ok=True) | |
| save_file( | |
| model.state_dict(), | |
| os.path.join(output_dir, "model.safetensors") | |
| ) | |
| config.save_pretrained(output_dir) | |