Instructions to use HappyBulingling/MyModel-pretrain with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use HappyBulingling/MyModel-pretrain with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="HappyBulingling/MyModel-pretrain", trust_remote_code=True) messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("HappyBulingling/MyModel-pretrain", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use HappyBulingling/MyModel-pretrain with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "HappyBulingling/MyModel-pretrain" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "HappyBulingling/MyModel-pretrain", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/HappyBulingling/MyModel-pretrain
- SGLang
How to use HappyBulingling/MyModel-pretrain with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "HappyBulingling/MyModel-pretrain" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "HappyBulingling/MyModel-pretrain", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "HappyBulingling/MyModel-pretrain" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "HappyBulingling/MyModel-pretrain", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use HappyBulingling/MyModel-pretrain with Docker Model Runner:
docker model run hf.co/HappyBulingling/MyModel-pretrain
Download attention.py from HappyBulingling/MyModel-pretrain: direct link, hf CLI and curl.
- Browser
- Download file 4.56 kB
-
https://huggingface.co/HappyBulingling/MyModel-pretrain/resolve/main/attention.py
- Command line
-
hf download hf://HappyBulingling/MyModel-pretrain/attention.py
-
curl -L -o attention.py https://huggingface.co/HappyBulingling/MyModel-pretrain/resolve/main/attention.py
4.56 kB
| """Grouped-query attention with q/k RMSNorm and SDPA.""" | |
| from __future__ import annotations | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| from .config import MyModelConfig | |
| from .norm import RMSNorm | |
| from .rope import apply_rotary_pos_emb | |
| def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: | |
| if n_rep == 1: | |
| return hidden_states | |
| batch, seq_len, num_kv_heads, head_dim = hidden_states.shape | |
| hidden_states = hidden_states[:, :, :, None, :].expand( | |
| batch, seq_len, num_kv_heads, n_rep, head_dim | |
| ) | |
| return hidden_states.reshape(batch, seq_len, num_kv_heads * n_rep, head_dim) | |
| def causal_mask( | |
| q_len: int, | |
| kv_len: int, | |
| device: torch.device, | |
| ) -> torch.Tensor: | |
| """Boolean mask for a query block aligned to the end of the KV cache.""" | |
| rows = torch.arange(q_len, device=device).unsqueeze(1) + (kv_len - q_len) | |
| cols = torch.arange(kv_len, device=device).unsqueeze(0) | |
| return cols <= rows | |
| class Attention(nn.Module): | |
| def __init__(self, config: MyModelConfig): | |
| super().__init__() | |
| self.num_heads = config.num_attention_heads | |
| self.num_kv_heads = config.num_key_value_heads | |
| self.head_dim = config.head_dim | |
| self.num_kv_groups = self.num_heads // self.num_kv_heads | |
| self.dropout = config.dropout | |
| self.q_proj = nn.Linear( | |
| config.hidden_size, | |
| self.num_heads * self.head_dim, | |
| bias=config.attention_bias, | |
| ) | |
| self.k_proj = nn.Linear( | |
| config.hidden_size, | |
| self.num_kv_heads * self.head_dim, | |
| bias=config.attention_bias, | |
| ) | |
| self.v_proj = nn.Linear( | |
| config.hidden_size, | |
| self.num_kv_heads * self.head_dim, | |
| bias=config.attention_bias, | |
| ) | |
| self.o_proj = nn.Linear( | |
| self.num_heads * self.head_dim, | |
| config.hidden_size, | |
| bias=config.attention_bias, | |
| ) | |
| self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity() | |
| self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity() | |
| def forward( | |
| self, | |
| hidden_states: torch.Tensor, | |
| cos: torch.Tensor, | |
| sin: torch.Tensor, | |
| past_key_value: tuple[torch.Tensor, torch.Tensor] | None = None, | |
| use_cache: bool = False, | |
| attention_mask: torch.Tensor | None = None, | |
| ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]: | |
| batch, seq_len, _ = hidden_states.shape | |
| query = self.q_proj(hidden_states).view(batch, seq_len, self.num_heads, self.head_dim) | |
| key = self.k_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim) | |
| value = self.v_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim) | |
| query = self.q_norm(query) | |
| key = self.k_norm(key) | |
| query, key = apply_rotary_pos_emb(query, key, cos, sin) | |
| if past_key_value is not None: | |
| key = torch.cat((past_key_value[0], key), dim=1) | |
| value = torch.cat((past_key_value[1], value), dim=1) | |
| present = (key, value) if use_cache else None | |
| kv_len = key.shape[1] | |
| key = repeat_kv(key, self.num_kv_groups).transpose(1, 2) | |
| value = repeat_kv(value, self.num_kv_groups).transpose(1, 2) | |
| query = query.transpose(1, 2) | |
| dropout_p = self.dropout if self.training else 0.0 | |
| if attention_mask is None and past_key_value is None: | |
| output = F.scaled_dot_product_attention( | |
| query, | |
| key, | |
| value, | |
| dropout_p=dropout_p, | |
| is_causal=True, | |
| ) | |
| else: | |
| mask = causal_mask(seq_len, kv_len, hidden_states.device) | |
| if attention_mask is not None: | |
| if attention_mask.shape != (batch, kv_len): | |
| raise ValueError( | |
| f"attention_mask must have shape {(batch, kv_len)}, " | |
| f"got {tuple(attention_mask.shape)}" | |
| ) | |
| mask = mask.unsqueeze(0).unsqueeze(0) & attention_mask[:, None, None, :].bool() | |
| output = F.scaled_dot_product_attention( | |
| query, | |
| key, | |
| value, | |
| attn_mask=mask, | |
| dropout_p=dropout_p, | |
| ) | |
| output = output.transpose(1, 2).reshape(batch, seq_len, -1) | |
| return self.o_proj(output), present | |