Instructions to use HappyBulingling/MyModel-sft with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use HappyBulingling/MyModel-sft with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="HappyBulingling/MyModel-sft", trust_remote_code=True) messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("HappyBulingling/MyModel-sft", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use HappyBulingling/MyModel-sft with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "HappyBulingling/MyModel-sft" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "HappyBulingling/MyModel-sft", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/HappyBulingling/MyModel-sft
- SGLang
How to use HappyBulingling/MyModel-sft with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "HappyBulingling/MyModel-sft" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "HappyBulingling/MyModel-sft", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "HappyBulingling/MyModel-sft" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "HappyBulingling/MyModel-sft", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use HappyBulingling/MyModel-sft with Docker Model Runner:
docker model run hf.co/HappyBulingling/MyModel-sft
Download model.py from HappyBulingling/MyModel-sft: direct link, hf CLI and curl.
- Browser
- Download file 11 kB
-
https://huggingface.co/HappyBulingling/MyModel-sft/resolve/main/model.py
- Command line
-
hf download hf://HappyBulingling/MyModel-sft/model.py
-
curl -L -o model.py https://huggingface.co/HappyBulingling/MyModel-sft/resolve/main/model.py
11 kB
| """MyModel backbone and causal language-model head.""" | |
| from __future__ import annotations | |
| import math | |
| from collections.abc import Sequence | |
| from typing import Iterable | |
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| from torch.utils.checkpoint import checkpoint | |
| from transformers import GenerationMixin, PreTrainedModel | |
| from transformers.modeling_outputs import CausalLMOutputWithPast | |
| from .block import Block | |
| from .config import MyModelConfig | |
| from .norm import RMSNorm | |
| from .rope import precompute_rope | |
| def chunked_cross_entropy( | |
| hidden_states: torch.Tensor, | |
| labels: torch.Tensor, | |
| lm_head: nn.Linear, | |
| chunk_size: int, | |
| ignore_index: int = -100, | |
| ) -> torch.Tensor: | |
| """Next-token cross entropy without materializing all logits at once.""" | |
| if hidden_states.shape[:2] != labels.shape: | |
| raise ValueError( | |
| f"hidden_states {tuple(hidden_states.shape[:2])} and labels " | |
| f"{tuple(labels.shape)} must agree on batch and sequence length" | |
| ) | |
| if chunk_size <= 0: | |
| raise ValueError("chunk_size must be positive") | |
| hidden = hidden_states[:, :-1, :] | |
| target = labels[:, 1:] | |
| total_loss = hidden_states.new_zeros(()) | |
| total_tokens = 0 | |
| for start in range(0, hidden.shape[1], chunk_size): | |
| chunk_hidden = hidden[:, start : start + chunk_size, :] | |
| chunk_target = target[:, start : start + chunk_size] | |
| logits = lm_head(chunk_hidden) | |
| total_loss = total_loss + F.cross_entropy( | |
| logits.reshape(-1, logits.shape[-1]), | |
| chunk_target.reshape(-1), | |
| ignore_index=ignore_index, | |
| reduction="sum", | |
| ) | |
| total_tokens += int((chunk_target != ignore_index).sum()) | |
| return total_loss / max(total_tokens, 1) | |
| class MyModelModel(nn.Module): | |
| def __init__(self, config: MyModelConfig): | |
| super().__init__() | |
| self.config = config | |
| self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) | |
| self.dropout = nn.Dropout(config.dropout) | |
| self.layers = nn.ModuleList([Block(config) for _ in range(config.num_hidden_layers)]) | |
| self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| cos, sin = precompute_rope( | |
| config.head_dim, | |
| config.max_position_embeddings, | |
| config.rope_theta, | |
| ) | |
| self.register_buffer("rope_cos", cos, persistent=False) | |
| self.register_buffer("rope_sin", sin, persistent=False) | |
| self.gradient_checkpointing = False | |
| def gradient_checkpointing_enable(self) -> None: | |
| self.gradient_checkpointing = True | |
| def gradient_checkpointing_disable(self) -> None: | |
| self.gradient_checkpointing = False | |
| def forward( | |
| self, | |
| input_ids: torch.Tensor, | |
| attention_mask: torch.Tensor | None = None, | |
| past_key_values: Iterable[tuple[torch.Tensor, torch.Tensor] | None] | None = None, | |
| use_cache: bool = False, | |
| ) -> tuple[torch.Tensor, tuple[tuple[torch.Tensor, torch.Tensor], ...] | None]: | |
| batch, seq_len = input_ids.shape | |
| if past_key_values is None: | |
| past_key_values = [None] * len(self.layers) | |
| else: | |
| past_key_values = list(past_key_values) | |
| if len(past_key_values) != len(self.layers): | |
| raise ValueError( | |
| f"expected {len(self.layers)} past key/value entries, " | |
| f"got {len(past_key_values)}" | |
| ) | |
| start_pos = 0 if past_key_values[0] is None else past_key_values[0][0].shape[1] | |
| if start_pos + seq_len > self.config.max_position_embeddings: | |
| raise ValueError( | |
| f"sequence length {start_pos + seq_len} exceeds " | |
| f"max_position_embeddings {self.config.max_position_embeddings}" | |
| ) | |
| hidden_states = self.dropout(self.embed_tokens(input_ids)) | |
| cos = self.rope_cos[start_pos : start_pos + seq_len] | |
| sin = self.rope_sin[start_pos : start_pos + seq_len] | |
| presents: list[tuple[torch.Tensor, torch.Tensor] | None] = [] | |
| for layer, past_key_value in zip(self.layers, past_key_values): | |
| if self.gradient_checkpointing and self.training: | |
| hidden_states, _ = checkpoint( | |
| layer, | |
| hidden_states, | |
| cos, | |
| sin, | |
| past_key_value, | |
| False, | |
| attention_mask, | |
| use_reentrant=False, | |
| ) | |
| presents.append(None) | |
| else: | |
| hidden_states, present = layer( | |
| hidden_states, | |
| cos, | |
| sin, | |
| past_key_value, | |
| use_cache, | |
| attention_mask, | |
| ) | |
| presents.append(present) | |
| hidden_states = self.norm(hidden_states) | |
| if not use_cache: | |
| return hidden_states, None | |
| return hidden_states, tuple(presents) | |
| class MyModelForCausalLM(PreTrainedModel, GenerationMixin): | |
| config_class = MyModelConfig | |
| _tied_weights_keys = ["lm_head.weight"] | |
| def __init__(self, config: MyModelConfig): | |
| super().__init__(config) | |
| self.model = MyModelModel(config) | |
| self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) | |
| if config.tie_word_embeddings: | |
| self.lm_head.weight = self.model.embed_tokens.weight | |
| self.post_init() | |
| self._scale_residual_projections() | |
| def _init_weights(self, module: nn.Module) -> None: | |
| if isinstance(module, nn.Linear): | |
| module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) | |
| if module.bias is not None: | |
| module.bias.data.zero_() | |
| elif isinstance(module, nn.Embedding): | |
| module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) | |
| def _scale_residual_projections(self) -> None: | |
| scale = 1.0 / math.sqrt(2 * self.config.num_hidden_layers) | |
| with torch.no_grad(): | |
| for layer in self.model.layers: | |
| layer.self_attn.o_proj.weight.mul_(scale) | |
| layer.mlp.down_proj.weight.mul_(scale) | |
| def gradient_checkpointing_enable(self) -> None: | |
| self.model.gradient_checkpointing_enable() | |
| def gradient_checkpointing_disable(self) -> None: | |
| self.model.gradient_checkpointing_disable() | |
| def forward( | |
| self, | |
| input_ids: torch.Tensor, | |
| attention_mask: torch.Tensor | None = None, | |
| past_key_values: Iterable[tuple[torch.Tensor, torch.Tensor] | None] | None = None, | |
| use_cache: bool = False, | |
| labels: torch.Tensor | None = None, | |
| chunk_size: int = 0, | |
| logits_to_keep: int = 0, | |
| **kwargs, | |
| ) -> CausalLMOutputWithPast: | |
| hidden_states, presents = self.model( | |
| input_ids, | |
| attention_mask=attention_mask, | |
| past_key_values=past_key_values, | |
| use_cache=use_cache, | |
| ) | |
| loss = None | |
| logits = None | |
| if labels is not None and chunk_size and chunk_size > 0: | |
| loss = chunked_cross_entropy(hidden_states, labels, self.lm_head, chunk_size) | |
| else: | |
| logits = self.lm_head(hidden_states) | |
| if labels is not None: | |
| shift_logits = logits[:, :-1, :] | |
| shift_labels = labels[:, 1:] | |
| loss = F.cross_entropy( | |
| shift_logits.reshape(-1, shift_logits.shape[-1]), | |
| shift_labels.reshape(-1), | |
| ignore_index=-100, | |
| ) | |
| if logits_to_keep > 0: | |
| logits = logits[:, -logits_to_keep:, :] | |
| return CausalLMOutputWithPast( | |
| loss=loss, | |
| logits=logits, | |
| past_key_values=presents, | |
| hidden_states=hidden_states, | |
| attentions=None, | |
| ) | |
| def generate( | |
| self, | |
| input_ids: torch.Tensor, | |
| max_new_tokens: int = 64, | |
| temperature: float = 1.0, | |
| top_k: int = 0, | |
| top_p: float = 1.0, | |
| eos_token_id: int | Sequence[int] | None = None, | |
| do_sample: bool = True, | |
| use_cache: bool = True, | |
| ) -> torch.Tensor: | |
| if temperature <= 0: | |
| raise ValueError("temperature must be positive") | |
| if eos_token_id is None: | |
| stop_ids = [int(self.config.eos_token_id)] | |
| im_end = getattr(self.config, "im_end_token_id", None) | |
| if im_end is not None: | |
| stop_ids.append(int(im_end)) | |
| elif isinstance(eos_token_id, int): | |
| stop_ids = [eos_token_id] | |
| else: | |
| stop_ids = [int(token_id) for token_id in eos_token_id] | |
| stop_ids = list(dict.fromkeys(stop_ids)) | |
| if not stop_ids: | |
| raise ValueError("eos_token_id must contain at least one token id") | |
| stop_tensor = torch.tensor( | |
| stop_ids, dtype=torch.long, device=input_ids.device | |
| ) | |
| fill_id = stop_ids[0] | |
| past_key_values = None | |
| finished = torch.zeros(input_ids.shape[0], dtype=torch.bool, device=input_ids.device) | |
| for _ in range(max_new_tokens): | |
| past_len = 0 if past_key_values is None else past_key_values[0][0].shape[1] | |
| outputs = self( | |
| input_ids[:, past_len:], | |
| past_key_values=past_key_values, | |
| use_cache=use_cache, | |
| ) | |
| logits = outputs.logits[:, -1, :] / temperature | |
| if top_k > 0: | |
| threshold = torch.topk(logits, min(top_k, logits.shape[-1]), dim=-1).values[..., -1:] | |
| logits = logits.masked_fill(logits < threshold, float("-inf")) | |
| if top_p < 1.0: | |
| sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1) | |
| cumulative = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1) | |
| remove = cumulative > top_p | |
| remove[..., 1:] = remove[..., :-1].clone() | |
| remove[..., 0] = False | |
| logits = logits.masked_fill(remove.scatter(1, sorted_indices, remove), float("-inf")) | |
| if do_sample: | |
| probs = torch.softmax(logits, dim=-1) | |
| next_token = torch.multinomial(probs, num_samples=1) | |
| else: | |
| next_token = torch.argmax(logits, dim=-1, keepdim=True) | |
| next_token = torch.where( | |
| finished.unsqueeze(-1), | |
| next_token.new_full((next_token.shape[0], 1), fill_id), | |
| next_token, | |
| ) | |
| input_ids = torch.cat((input_ids, next_token), dim=-1) | |
| past_key_values = outputs.past_key_values if use_cache else None | |
| finished |= (next_token == stop_tensor).any(dim=-1).reshape(-1) | |
| if finished.all(): | |
| break | |
| return input_ids | |