Instructions to use HappyBulingling/MyModel-sft with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use HappyBulingling/MyModel-sft with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="HappyBulingling/MyModel-sft", trust_remote_code=True) messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("HappyBulingling/MyModel-sft", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use HappyBulingling/MyModel-sft with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "HappyBulingling/MyModel-sft" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "HappyBulingling/MyModel-sft", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/HappyBulingling/MyModel-sft
- SGLang
How to use HappyBulingling/MyModel-sft with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "HappyBulingling/MyModel-sft" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "HappyBulingling/MyModel-sft", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "HappyBulingling/MyModel-sft" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "HappyBulingling/MyModel-sft", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use HappyBulingling/MyModel-sft with Docker Model Runner:
docker model run hf.co/HappyBulingling/MyModel-sft
File size: 4,564 Bytes
72f5707 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | """Grouped-query attention with q/k RMSNorm and SDPA."""
from __future__ import annotations
import torch
import torch.nn.functional as F
from torch import nn
from .config import MyModelConfig
from .norm import RMSNorm
from .rope import apply_rotary_pos_emb
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
if n_rep == 1:
return hidden_states
batch, seq_len, num_kv_heads, head_dim = hidden_states.shape
hidden_states = hidden_states[:, :, :, None, :].expand(
batch, seq_len, num_kv_heads, n_rep, head_dim
)
return hidden_states.reshape(batch, seq_len, num_kv_heads * n_rep, head_dim)
def causal_mask(
q_len: int,
kv_len: int,
device: torch.device,
) -> torch.Tensor:
"""Boolean mask for a query block aligned to the end of the KV cache."""
rows = torch.arange(q_len, device=device).unsqueeze(1) + (kv_len - q_len)
cols = torch.arange(kv_len, device=device).unsqueeze(0)
return cols <= rows
class Attention(nn.Module):
def __init__(self, config: MyModelConfig):
super().__init__()
self.num_heads = config.num_attention_heads
self.num_kv_heads = config.num_key_value_heads
self.head_dim = config.head_dim
self.num_kv_groups = self.num_heads // self.num_kv_heads
self.dropout = config.dropout
self.q_proj = nn.Linear(
config.hidden_size,
self.num_heads * self.head_dim,
bias=config.attention_bias,
)
self.k_proj = nn.Linear(
config.hidden_size,
self.num_kv_heads * self.head_dim,
bias=config.attention_bias,
)
self.v_proj = nn.Linear(
config.hidden_size,
self.num_kv_heads * self.head_dim,
bias=config.attention_bias,
)
self.o_proj = nn.Linear(
self.num_heads * self.head_dim,
config.hidden_size,
bias=config.attention_bias,
)
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity()
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity()
def forward(
self,
hidden_states: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
past_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
use_cache: bool = False,
attention_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
batch, seq_len, _ = hidden_states.shape
query = self.q_proj(hidden_states).view(batch, seq_len, self.num_heads, self.head_dim)
key = self.k_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim)
value = self.v_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim)
query = self.q_norm(query)
key = self.k_norm(key)
query, key = apply_rotary_pos_emb(query, key, cos, sin)
if past_key_value is not None:
key = torch.cat((past_key_value[0], key), dim=1)
value = torch.cat((past_key_value[1], value), dim=1)
present = (key, value) if use_cache else None
kv_len = key.shape[1]
key = repeat_kv(key, self.num_kv_groups).transpose(1, 2)
value = repeat_kv(value, self.num_kv_groups).transpose(1, 2)
query = query.transpose(1, 2)
dropout_p = self.dropout if self.training else 0.0
if attention_mask is None and past_key_value is None:
output = F.scaled_dot_product_attention(
query,
key,
value,
dropout_p=dropout_p,
is_causal=True,
)
else:
mask = causal_mask(seq_len, kv_len, hidden_states.device)
if attention_mask is not None:
if attention_mask.shape != (batch, kv_len):
raise ValueError(
f"attention_mask must have shape {(batch, kv_len)}, "
f"got {tuple(attention_mask.shape)}"
)
mask = mask.unsqueeze(0).unsqueeze(0) & attention_mask[:, None, None, :].bool()
output = F.scaled_dot_product_attention(
query,
key,
value,
attn_mask=mask,
dropout_p=dropout_p,
)
output = output.transpose(1, 2).reshape(batch, seq_len, -1)
return self.o_proj(output), present
|