shibatch commited on
Commit
12e1583
·
verified ·
1 Parent(s): 24bbdc6

Upload README.md with huggingface_hub

Browse files
Files changed (1) hide show
  1. README.md +27 -13
README.md CHANGED
@@ -97,35 +97,49 @@ To process the GQA MoE execution graph and evaluate dynamic expert routing direc
97
 
98
  ```python
99
  import torch
100
- from transformers import AutoTokenizer, AutoModelForCausalLM
 
 
101
 
 
102
  repo_id = "shibatch/tinymoeja2m"
103
 
104
- print("Loading Japanese MoE GQA configuration and tokenizer layers...")
105
- tokenizer = AutoTokenizer.from_pretrained(repo_id, subfolder="hf")
106
- model = AutoModelForCausalLM.from_pretrained(repo_id, subfolder="hf")
 
 
 
 
 
 
107
 
108
  device = "cuda" if torch.cuda.is_available() else ("xpu" if torch.xpu.is_available() else "cpu")
109
  model = model.to(device)
110
  model.eval()
111
 
 
112
  prompt = "トムとリリーは"
113
- inputs = tokenizer(prompt, return_tensors="pt").to(device)
 
114
 
115
- print("Running inference loop (Validating 4:1 GQA & Top-2 MoE Kernels)...")
116
  with torch.no_grad():
117
- outputs = model.generate(
118
- **inputs,
119
- max_length=64,
120
- do_sample=False
 
 
 
121
  )
122
-
123
- generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
 
124
 
125
  print("\n--- Inference Test Result ---")
126
  print("Prompt :", prompt)
127
  print("Generated:", generated_text)
128
-
129
  ```
130
 
131
  ---
 
97
 
98
  ```python
99
  import torch
100
+ import sentencepiece as spm
101
+ from transformers import MixtralForCausalLM
102
+ from huggingface_hub import hf_hub_download
103
 
104
+ # Define target repository identity
105
  repo_id = "shibatch/tinymoeja2m"
106
 
107
+ print("Downloading and caching specialized tokenizer layer...")
108
+ # Fetch tokenizer.model file automatically from Hugging Face Hub
109
+ tokenizer_file = hf_hub_download(repo_id=repo_id, subfolder="hf", filename="tokenizer.model")
110
+
111
+ sp = spm.SentencePieceProcessor()
112
+ sp.Load(tokenizer_file)
113
+
114
+ print("Downloading and loading Mixtral-based 2M MoE model weights...")
115
+ model = MixtralForCausalLM.from_pretrained(repo_id, subfolder="hf")
116
 
117
  device = "cuda" if torch.cuda.is_available() else ("xpu" if torch.xpu.is_available() else "cpu")
118
  model = model.to(device)
119
  model.eval()
120
 
121
+ # Prompt text utilizing vocabulary subsets
122
  prompt = "トムとリリーは"
123
+ input_ids = [1] + sp.EncodeAsIds(prompt) # Explicitly prepend BOS (1)
124
+ input_tensor = torch.tensor([input_ids]).to(device)
125
 
126
+ print("Executing text generation loop (Validating 4:1 GQA & Top-2 MoE Kernels)...")
127
  with torch.no_grad():
128
+ output_ids = model.generate(
129
+ input_tensor,
130
+ max_length=64,
131
+ do_sample=False,
132
+ pad_token_id=2,
133
+ bos_token_id=1,
134
+ eos_token_id=2
135
  )
136
+
137
+ generated_ids = output_ids[0].cpu().tolist()
138
+ generated_text = sp.DecodeIds(generated_ids)
139
 
140
  print("\n--- Inference Test Result ---")
141
  print("Prompt :", prompt)
142
  print("Generated:", generated_text)
 
143
  ```
144
 
145
  ---