Add files using upload-large-folder tool

README.md

@@ -0,0 +1,115 @@
+---
+license: apache-2.0
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+pipeline_tag: text-generation
+library_name: tensorrt-llm
+tags:
+- qwen2
+- qwen
+- tensorrt-llm
+- text-generation
+- fp16
+- checkpoint
+---
+
+# Qwen2.5-1.5B-Instruct TensorRT-LLM Checkpoint (FP16)
+
+This repository contains a community-converted TensorRT-LLM checkpoint for [`Qwen/Qwen2.5-1.5B-Instruct`](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+
+It is a TensorRT-LLM **checkpoint-format** repository, not a prebuilt engine. The intent is to let you download the checkpoint from Hugging Face and build an engine locally for your own GPU and TensorRT-LLM version.
+
+## Details
+
+- Base model: `Qwen/Qwen2.5-1.5B-Instruct`
+- License: `apache-2.0`
+- TensorRT-LLM version used for conversion: `1.2.0rc6`
+- Checkpoint dtype: `float16`
+- Tensor parallel size: `1`
+- Checkpoint files:
+  - `config.json`
+  - `rank0.safetensors`
+  - tokenizer and generation files copied from the upstream Hugging Face model
+
+## Files
+
+- `config.json`: TensorRT-LLM checkpoint config
+- `rank0.safetensors`: TensorRT-LLM checkpoint weights
+- `generation_config.json`: upstream generation config
+- `tokenizer.json`: upstream tokenizer
+- `tokenizer_config.json`: upstream tokenizer config
+- `merges.txt`: upstream merges file
+- `vocab.json`: upstream vocabulary
+
+## Build
+
+Build an engine locally with TensorRT-LLM:
+
+```bash
+huggingface-cli download Shoolife/Qwen2.5-1.5B-Instruct-TensorRT-LLM-Checkpoint-FP16 --local-dir ./checkpoint
+
+trtllm-build \
+  --checkpoint_dir ./checkpoint \
+  --output_dir ./engine \
+  --gemm_plugin float16 \
+  --gpt_attention_plugin float16 \
+  --max_batch_size 1 \
+  --max_input_len 512 \
+  --max_seq_len 1024 \
+  --max_num_tokens 256 \
+  --workers 1 \
+  --monitor_memory
+```
+
+## Conversion
+
+This checkpoint was produced from the upstream model with TensorRT-LLM Qwen conversion tooling:
+
+```bash
+python convert_checkpoint.py \
+  --model_dir ./Qwen2.5-1.5B-Instruct \
+  --output_dir ./checkpoint_fp16 \
+  --dtype float16
+```
+
+## Validation
+
+The checkpoint was validated by building a local engine and running inference on:
+
+- GPU: `NVIDIA GeForce RTX 5070 Laptop GPU`
+- Runtime: `TensorRT-LLM 1.2.0rc6`
+
+Smoke-test prompt:
+
+```text
+Explain the four basic arithmetic operations in one short sentence each.
+```
+
+Observed response:
+
+```text
+Addition: Combining two or more numbers to find their total sum.
+Subtraction: Removing one number from another to find the difference.
+Multiplication: Repeated addition of a number to itself a certain number of times.
+Division: Splitting a number into equal parts or groups.
+```
+
+## Benchmark Snapshot
+
+Local single-GPU measurements from the built engine on `RTX 5070 Laptop GPU`, using TensorRT-LLM synthetic fixed-length requests, `20` requests per profile, `2` warmup requests, and `concurrency=1`.
+
+| Profile | Input | Output | TTFT | TPOT | Output tok/s | Avg latency |
+|---|---:|---:|---:|---:|---:|---:|
+| `tiny_16_32` | 16 | 32 | `11.78 ms` | `9.43 ms` | `105.22` | `304.11 ms` |
+| `short_chat_42_64` | 42 | 64 | `12.22 ms` | `9.44 ms` | `105.48` | `606.75 ms` |
+| `balanced_128_128` | 128 | 128 | `13.41 ms` | `9.45 ms` | `105.49` | `1213.30 ms` |
+| `long_prompt_192_64` | 192 | 64 | `16.54 ms` | `9.45 ms` | `104.57` | `612.02 ms` |
+| `long_generation_42_192` | 42 | 192 | `12.35 ms` | `9.45 ms` | `105.70` | `1816.42 ms` |
+
+These numbers are local measurements from one machine and should be treated as reference values, not portability guarantees.
+
+## Notes
+
+- This is not an official Qwen or NVIDIA release.
+- This repository does not include a prebuilt TensorRT engine.
+- Engine compatibility and performance depend on your GPU, driver, CUDA, TensorRT, and TensorRT-LLM versions.
+

config.json

@@ -0,0 +1,73 @@
+{
+    "mlp_bias": false,
+    "attn_bias": true,
+    "rotary_base": 1000000.0,
+    "rotary_scaling": null,
+    "disable_weight_only_quant_plugin": false,
+    "num_labels": 1,
+    "use_logn_attn": false,
+    "mlp_only_layers": [],
+    "decoder_sparse_step": 1,
+    "moe": {
+        "num_experts": 0,
+        "shared_expert_intermediate_size": 0,
+        "top_k": 0,
+        "normalization_mode": 0,
+        "sparse_mixer_epsilon": 0.01,
+        "tp_mode": 0,
+        "device_limited_n_group": 0,
+        "device_limited_topk_group": 0,
+        "device_limited_routed_scaling_factor": 1.0
+    },
+    "architecture": "Qwen2ForCausalLM",
+    "dtype": "float16",
+    "vocab_size": 151936,
+    "hidden_size": 1536,
+    "num_hidden_layers": 28,
+    "num_attention_heads": 12,
+    "hidden_act": "silu",
+    "logits_dtype": "float32",
+    "norm_epsilon": 1e-06,
+    "runtime_defaults": null,
+    "position_embedding_type": "rope_gpt_neox",
+    "num_key_value_heads": 2,
+    "intermediate_size": 8960,
+    "max_position_embeddings": 32768,
+    "mapping": {
+        "world_size": 1,
+        "gpus_per_node": 8,
+        "cp_size": 1,
+        "tp_size": 1,
+        "pp_size": 1,
+        "moe_tp_size": 1,
+        "moe_cluster_size": 1,
+        "moe_ep_size": 1,
+        "attn_tp_size": -1,
+        "attn_cp_size": -1,
+        "cp_config": {},
+        "enable_attention_dp": false,
+        "enable_lm_head_tp_in_adp": false
+    },
+    "quantization": {
+        "quant_algo": null,
+        "kv_cache_quant_algo": null,
+        "group_size": 128,
+        "smoothquant_val": 0.5,
+        "clamp_val": null,
+        "use_meta_recipe": false,
+        "has_zero_point": false,
+        "pre_quant_scale": false,
+        "exclude_modules": null,
+        "mamba_ssm_cache_dtype": null
+    },
+    "use_parallel_embedding": false,
+    "embedding_sharding_dim": 0,
+    "head_size": 128,
+    "qk_layernorm": false,
+    "rotary_embedding_dim": 128,
+    "seq_length": 8192,
+    "qwen_type": "qwen2",
+    "moe_intermediate_size": 0,
+    "moe_shared_expert_intermediate_size": 0,
+    "tie_word_embeddings": true
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "pad_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_p": 0.8,
+  "top_k": 20,
+  "transformers_version": "4.37.0"
+}

rank0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:261d01b5eecbd188287caef1a81365806035a72005c295c90991bf087feeb0ec
+size 3587757176

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}