Upload folder using huggingface_hub

README.md

@@ -0,0 +1,44 @@
+---
+base_model: skt/A.X-4.0
+quantized_by: tmfi-us
+license: other
+license_name: qwen
+license_link: https://huggingface.co/skt/A.X-4.0/blob/main/LICENSE
+language:
+  - en
+  - ko
+pipeline_tag: text-generation
+---
+
+## About
+
+FP8 w8a8 dynamic quants of [skt/A.X-4.0](https://huggingface.co/skt/A.X-4.0).
+
+Used following Python script with [llmcompressor](https://github.com/vllm-project/llm-compressor) to generate:
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from llmcompressor.transformers import oneshot
+from llmcompressor.modifiers.quantization import QuantizationModifier
+
+MODEL_ID = 'skt/A.X-4.0'
+
+model = AutoModelForCausalLM.from_pretrained(
+  MODEL_ID, device_map="auto", torch_dtype="auto",
+)
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+
+# Configure the simple PTQ quantization
+recipe = QuantizationModifier(
+  targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"])
+
+# Apply the quantization algorithm.
+oneshot(model=model, recipe=recipe)
+
+# Save the model.
+SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-Dynamic"
+model.save_pretrained(SAVE_DIR)
+tokenizer.save_pretrained(SAVE_DIR)
+```
+
+Quantization recipe can be found in [`recipe.yaml`](recipe.yaml)

config.json

@@ -0,0 +1,75 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 29568,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": 8,
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "input_activations": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": true,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": null,
+          "observer_kwargs": {},
+          "strategy": "token",
+          "symmetric": true,
+          "type": "float"
+        },
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": "minmax",
+          "observer_kwargs": {},
+          "strategy": "channel",
+          "symmetric": true,
+          "type": "float"
+        }
+      }
+    },
+    "format": "float-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 2.0,
+    "original_max_position_embeddings": 131072,
+    "rope_type": "yarn",
+    "type": "yarn"
+  },
+  "rope_theta": 5000000,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.3",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 102400
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "bos_token_id": 0,
+  "do_sample": true,
+  "eos_token_id": 27,
+  "pad_token_id": 1,
+  "temperature": 0.6,
+  "top_p": 0.95,
+  "transformers_version": "4.50.3"
+}

model-00001-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ff5f4a40e60e24b80019f3a0a613e55f133fba3dc97bf970cf008fa2b64107e
+size 4946991480

model-00002-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:160b092a1fe60a9efa374dbe21a289bb9cea1e35c930a582ff0aeb10a3b2debe
+size 4782749264

model-00003-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3db661d3323052592d4cc9d3daa029ed2579747b5a4a7c59b45fe74fa58e5b02
+size 4873986008

model-00004-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bc89828bc796c08941d26992557db57b3f9d0077ccabda73c38fda60eff4981
+size 4782749368

model-00005-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dbeac4c1a546cd4c11b5d7e4c1b814456989940d9457db74fe036a9e32ecf22
+size 4873986032

model-00006-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30868576ff59f107e94176d37df7ee1281e36aece37d1620231947ce75bdb466
+size 4782749368

model-00007-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89b5de72a28944c056c6d70e7f65c11ec05b3c96715146f70a9b5846b271cd1e
+size 4873986032

model-00008-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb7bcd2dd2cce648eb22bcef6c5fb69b2509bbc078747d16aed34e6791b07ecf
+size 4782749368

model-00009-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:145b0885ac498c8d8f308055a38768e078487555da39360aa671d12cf9698bc9
+size 4873986032

model-00010-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9f8b1b8aca84f268982f2bc00fad5b5907ce5c8c1df79f55d6d6aba2c0cbf0d
+size 4782749368

model-00011-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c99983766f56d2c1e1559549308c953d832edc9543fd1659f7ba6976e9b165d
+size 4873986032

model-00012-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62433ea9c1dae60a130ae51ffea4912e8d9b1b5476809f0b13a13e057cacd3ba
+size 4782749368

model-00013-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a90e6e27460359b7662bff013f8654e4f6294c41ef785c6574e016a2234be9f
+size 4873986032

model-00014-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24c2ef4f85dd0a9abf8db63058ea0cfa3f291fcca8d77470a3dac1bdeff6e480
+size 4782749368

model-00015-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d6da2c3d3d1cc5923d33a1c44d016d86dbd9e4bf65e691793b4091416466cc7
+size 4238388104

model-00016-of-00016.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e905e870c873a9672de4e03f4843de19575f00308f003994e5860b1cb8b88771
+size 1677721728

recipe.yaml

@@ -0,0 +1,6 @@
+default_stage:
+  default_modifiers:
+    QuantizationModifier:
+      targets: [Linear]
+      ignore: [lm_head]
+      scheme: FP8_DYNAMIC

special_tokens_map.json

@@ -0,0 +1,90 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|pad|>",
+    "<|unk|>",
+    "<|sep|>",
+    "<|mask|>",
+    "<|cls|>",
+    "<|image|>",
+    "<|audio|>",
+    "<|user|>",
+    "<|system|>",
+    "<|assistant|>",
+    "<|extra_id_0|>",
+    "<|extra_id_1|>",
+    "<|extra_id_2|>",
+    "<|extra_id_3|>",
+    "<|extra_id_4|>",
+    "<|extra_id_5|>",
+    "<|extra_id_6|>",
+    "<|extra_id_7|>",
+    "<|extra_id_8|>",
+    "<|extra_id_9|>",
+    "<|extra_id_10|>",
+    "<|extra_id_11|>",
+    "<|extra_id_12|>",
+    "<|extra_id_13|>",
+    "<|im_start|>",
+    "<|im_sep|>",
+    "<|im_end|>",
+    "<|resident_reg|>",
+    "<|foreigner_reg|>",
+    "<|business_reg|>",
+    "<|credit_card|>",
+    "<|passport|>",
+    "<|driver_license|>",
+    "<|telephone|>",
+    "<|health_insurance|>",
+    "<|bank_account|>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<|cls|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<|mask|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "<|sep|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|unk|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,391 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|unk|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<|mask|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<|cls|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<|image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<|audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "19": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "20": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "21": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "22": {
+      "content": "<|extra_id_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "23": {
+      "content": "<|extra_id_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24": {
+      "content": "<|extra_id_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "26": {
+      "content": "<|im_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "27": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28": {
+      "content": "<|resident_reg|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "29": {
+      "content": "<|foreigner_reg|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30": {
+      "content": "<|business_reg|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "31": {
+      "content": "<|credit_card|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<|passport|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "33": {
+      "content": "<|driver_license|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "34": {
+      "content": "<|telephone|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "35": {
+      "content": "<|health_insurance|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "36": {
+      "content": "<|bank_account|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "37": {
+      "content": "</tool_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<tool_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|pad|>",
+    "<|unk|>",
+    "<|sep|>",
+    "<|mask|>",
+    "<|cls|>",
+    "<|image|>",
+    "<|audio|>",
+    "<|user|>",
+    "<|system|>",
+    "<|assistant|>",
+    "<|extra_id_0|>",
+    "<|extra_id_1|>",
+    "<|extra_id_2|>",
+    "<|extra_id_3|>",
+    "<|extra_id_4|>",
+    "<|extra_id_5|>",
+    "<|extra_id_6|>",
+    "<|extra_id_7|>",
+    "<|extra_id_8|>",
+    "<|extra_id_9|>",
+    "<|extra_id_10|>",
+    "<|extra_id_11|>",
+    "<|extra_id_12|>",
+    "<|extra_id_13|>",
+    "<|im_start|>",
+    "<|im_sep|>",
+    "<|im_end|>",
+    "<|resident_reg|>",
+    "<|foreigner_reg|>",
+    "<|business_reg|>",
+    "<|credit_card|>",
+    "<|passport|>",
+    "<|driver_license|>",
+    "<|telephone|>",
+    "<|health_insurance|>",
+    "<|bank_account|>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "chat_template": "{%- if tools is iterable and tools | length > 0 %}\n                    {{- '<|im_start|><|system|>'}}\n                    {{- '당신은 도구 호출 기능을 갖춘 유용한 도우미입니다. 사용자의 요청을 처리하기 위해서 필요한 도구가 주어진 목록에 있는 경우 도구 호출로 응답하세요.\n필요한 도구가 목록에 없는 경우에는 도구 호출 없이 사용자가 요구한 정보를 제공하세요.\n필요한 도구가 목록에 있지만 해당 도구를 호출하는데 필요한 argument 정보가 부족한 경우 해당 정보를 사용자에게 요청하세요.\n사용자의 요청을 처리하기 위해 여러번 도구를 호출할 수 있어야 합니다.\n도구 호출 이후 도구 실행 결과를 입력으로 받으면 해당 결과를 활용하여 답변을 생성하세요.\n\n다음은 접근할 수 있는 도구들의 목록 입니다:\n<tools>\n'}}\n                    {%- for t in tools %}\n                        {{- t | tojson }}\n                        {{- '\n' }}\n                    {%- endfor %}\n                    {{- '</tools>' }}\n                    {{- '\n\n도구를 호출하려면 아래의 JSON으로 응답하세요.\n도구 호출 형식: <tool_call>{\"name\": 도구 이름, \"arguments\": dictionary 형태의 도구 인자값}</tool_call>' }}\n                    {{- '<|im_end|>' }}\n                {%- endif %}\n        \n                {%- for message in messages %}\n                    {%- if message.role == 'system' %}\n                        {{- '<|im_start|><|system|>' + message.content + '<|im_end|>'}}\n                    {%- elif message.role == 'user' %}\n                        {{- '<|im_start|><|user|>' + message.content + '<|im_end|>'}}\n                    {%- elif message.role == 'assistant' %}\n                        {{- '<|im_start|><|assistant|>'}}\n                        {%- set content = '' %}\n                        {%- if message.content is defined %}\n                            {%- set content = message.content %}\n                        {%- endif %}\n                        \n                {%- if add_generation_prompt and not (message.reasoning_content is defined and message.reasoning_content is not none) %}\n                    {%- if '</think>' in message.content %}\n                        {%- set content = message.content.split('</think>'.strip())[-1].lstrip('\\n') %}\n                    {%- endif %}\n                {%- endif %}\n                \n                        {{- content}}\n                        {%- if message.tool_calls is defined %}\n                            {%- for tool_call in message.tool_calls %}\n                                {%- if tool_call.function is defined %}\n                                    {%- set tool_call = tool_call.function %}\n                                {%- endif %}\n                                {{- '<tool_call>' }}\n                                {{- '{' }}\n                                {{- '\"name\": \"' }}\n                                {{- tool_call.name }}\n                                {{- '\"' }}\n                                {%- if tool_call.arguments is defined %}\n                                    {{- ', ' }}\n                                    {{- '\"arguments\": ' }}\n                                    {{- tool_call.arguments|tojson }}\n                                {%- endif %}\n                                {{- '}' }}\n                                {{- '</tool_call>' }}\n                            {%- endfor %}\n                        {%- endif %}\n                        {{- '<|im_end|>'}}\n        \n                    {%- elif message.role == 'tool' %}\n                        {{- '<|im_start|><|extra_id_13|><tool_output>' + message.content + '</tool_output><|im_end|>'}}\n                    {%- endif %}\n                {%- endfor %}\n                \n                    {%- if add_generation_prompt %}\n                        {{- '<|im_start|><|assistant|>' }}\n                    {%- endif %}",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<|cls|>",
+  "eod_token": "<|endoftext|>",
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "mask_token": "<|mask|>",
+  "max_length": 7680,
+  "model_max_length": 262144,
+  "pad_token": "<|pad|>",
+  "padding_side": "right",
+  "sep_token": "<|sep|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": "<|unk|>",
+  "vocab_size": 102400
+}