add model files for mlc fotmat

Signed-off-by: David Wu <ktprms23@gmail.com>

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|PAD_TOKEN|>": 49152
+}

mlc-chat-config.json

@@ -0,0 +1,81 @@
+{
+  "version": "0.1.0",
+  "model_type": "llama",
+  "quantization": "q4f16_1",
+  "model_config": {
+    "hidden_size": 960,
+    "intermediate_size": 2560,
+    "num_attention_heads": 15,
+    "num_hidden_layers": 32,
+    "rms_norm_eps": 1e-05,
+    "vocab_size": 49153,
+    "tie_word_embeddings": true,
+    "position_embedding_base": 100000,
+    "rope_scaling": null,
+    "context_window_size": 8192,
+    "prefill_chunk_size": 8192,
+    "num_key_value_heads": 5,
+    "head_dim": 64,
+    "tensor_parallel_shards": 1,
+    "pipeline_parallel_stages": 1,
+    "max_batch_size": 128,
+    "disaggregation": false
+  },
+  "vocab_size": 49153,
+  "context_window_size": 8192,
+  "sliding_window_size": -1,
+  "prefill_chunk_size": 8192,
+  "attention_sink_size": -1,
+  "tensor_parallel_shards": 1,
+  "pipeline_parallel_stages": 1,
+  "temperature": 1.0,
+  "presence_penalty": 0.0,
+  "frequency_penalty": 0.0,
+  "repetition_penalty": 1.0,
+  "top_p": 1.0,
+  "tokenizer_files": [
+    "tokenizer.json",
+    "vocab.json",
+    "merges.txt",
+    "added_tokens.json",
+    "tokenizer_config.json"
+  ],
+  "tokenizer_info": {
+    "token_postproc_method": "byte_level",
+    "prepend_space_in_encode": false,
+    "strip_space_in_decode": false
+  },
+  "conv_template": {
+    "name": "chatml",
+    "system_template": "<|im_start|>system\n{system_message}<|im_end|>\n",
+    "system_message": "A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.",
+    "system_prefix_token_ids": null,
+    "add_role_after_system_message": true,
+    "roles": {
+      "user": "<|im_start|>user",
+      "assistant": "<|im_start|>assistant"
+    },
+    "role_templates": {
+      "user": "{user_message}",
+      "assistant": "{assistant_message}",
+      "tool": "{tool_message}"
+    },
+    "messages": [],
+    "seps": [
+      "<|im_end|>\n"
+    ],
+    "role_content_sep": "\n",
+    "role_empty_sep": "\n",
+    "stop_str": [
+      "<|im_end|>"
+    ],
+    "stop_token_ids": [
+      2
+    ],
+    "function_string": "",
+    "use_function_calling": false
+  },
+  "pad_token_id": 49152,
+  "bos_token_id": 1,
+  "eos_token_id": 2
+}

params_shard_0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eedd21484a4b24f8ce481e52f866e6136cba45bc4c2d85c51bc90421e1ddb601
+size 33460380

params_shard_1.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78bd4b91d545136a648ca505a38ec7cdd609d43bfb8b7d76b2c1f2e9e5d6b308
+size 33200640

params_shard_2.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dcdd84fd6a74a541312e4796815541553dee36cf20c496a959e3877b889313f
+size 33200640

params_shard_3.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05c2b23d802b77511a2165480280f2054c4e954e70e372865344c7433679e501
+size 33200640

params_shard_4.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49dd334166efc5868c65e477fc45e5bf3e0dc5139994a945910490ab14c4e854
+size 33200640

params_shard_5.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11a07a24efe080216ee92f6bf414773b8aba8c691f2ade9c899bc755bb8d62e5
+size 33200640

params_shard_6.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15d8afc5ad608aaf769295cc926bf8d76e3ca769492b9420dd0d16a9c8491e2a
+size 4151040

run_model.py

@@ -0,0 +1,42 @@
+from mlc_llm import MLCEngine
+
+# Create engine
+model = "."
+engine = MLCEngine(model, mode="interactive")
+
+messages = []
+
+print("輸入exit以結束")
+while True:
+    user_input = input("你：").strip()
+    if user_input.lower() == "exit":
+        break
+    
+    messages.append( {"role": "user", "content": user_input} )
+    response_string = ""
+    s_tmps = []
+    # Run chat completion in OpenAI API.
+#    response = await engine.chat.completions.create(
+    for response in engine.chat.completions.create(
+        messages=messages,
+        #messages=[{"role": "user", "content": user_input}],
+        model=model,
+        stream=True,
+    ):
+#    )
+     #   s_tmps = []
+        for choice in response.choices:
+            tmp = choice.delta.content
+            print(tmp, end="", flush=True)
+            s_tmps.append(tmp)
+    response_string = "".join(s_tmps)
+    messages.append( {"role": "assistant", "content": response_string} )
+    #print(response)
+    #response = stmps
+    #messages.append( {"role": "assistant", "content": response} )
+    print("\n")
+
+#    print(messages)
+
+
+engine.terminate()

tokenizer_config.json

@@ -0,0 +1,172 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "24211": {
+      "content": "�",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49152": {
+      "content": "<|PAD_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|PAD_TOKEN|>",
+  "padding_side": "left",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "�",
+  "vocab_size": 49152
+}