Initial commit

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[PAD]": 49152
+}

logs.txt

@@ -0,0 +1,105 @@
+/home/junrushao/micromamba/envs/python311/bin/python -m mlc_chat gen_config /home/junrushao/tmp/tmp5aeqhx5w/repo --quantization q4f32_1 --conv-template wizard_coder_or_math --output /home/junrushao/tmp/tmpblhqsgfd --context-window-size 8192
+[2023-12-29 13:45:49] INFO auto_config.py:115: Found model configuration: /home/junrushao/tmp/tmp5aeqhx5w/repo/config.json
+[2023-12-29 13:45:49] INFO auto_config.py:151: Found model type: gpt_bigcode. Use `--model-type` to override.
+[2023-12-29 13:45:49] INFO gpt_bigcode_model.py:41: context_window_size not found in config.json. Falling back to n_positions (8192)
+[2023-12-29 13:45:49] INFO flags_model_config_override.py:63: Default prefill_chunk_size to context_window_size (8192) because it is not provided
+[2023-12-29 13:45:49] INFO flags_model_config_override.py:112: Overriding context_window_size from 8192 to 8192
+[2023-12-29 13:45:49] INFO flags_model_config_override.py:112: Overriding prefill_chunk_size from 8192 to 8192
+[2023-12-29 13:45:49] INFO gen_config.py:115: [generation_config.json] Setting bos_token_id: 0
+[2023-12-29 13:45:49] INFO gen_config.py:115: [generation_config.json] Setting eos_token_id: 0
+[2023-12-29 13:45:49] INFO gen_config.py:129: Not found tokenizer config: /home/junrushao/tmp/tmp5aeqhx5w/repo/tokenizer.model
+[2023-12-29 13:45:49] INFO gen_config.py:127: Found tokenizer config: /home/junrushao/tmp/tmp5aeqhx5w/repo/tokenizer.json. Copying to /home/junrushao/tmp/tmpblhqsgfd/tokenizer.json
+[2023-12-29 13:45:49] INFO gen_config.py:127: Found tokenizer config: /home/junrushao/tmp/tmp5aeqhx5w/repo/vocab.json. Copying to /home/junrushao/tmp/tmpblhqsgfd/vocab.json
+[2023-12-29 13:45:49] INFO gen_config.py:127: Found tokenizer config: /home/junrushao/tmp/tmp5aeqhx5w/repo/merges.txt. Copying to /home/junrushao/tmp/tmpblhqsgfd/merges.txt
+[2023-12-29 13:45:49] INFO gen_config.py:127: Found tokenizer config: /home/junrushao/tmp/tmp5aeqhx5w/repo/added_tokens.json. Copying to /home/junrushao/tmp/tmpblhqsgfd/added_tokens.json
+[2023-12-29 13:45:49] INFO gen_config.py:127: Found tokenizer config: /home/junrushao/tmp/tmp5aeqhx5w/repo/tokenizer_config.json. Copying to /home/junrushao/tmp/tmpblhqsgfd/tokenizer_config.json
+[2023-12-29 13:45:49] INFO gen_config.py:69: [System default] Setting pad_token_id: 0
+[2023-12-29 13:45:49] INFO gen_config.py:69: [System default] Setting temperature: 0.7
+[2023-12-29 13:45:49] INFO gen_config.py:69: [System default] Setting repetition_penalty: 1.0
+[2023-12-29 13:45:49] INFO gen_config.py:69: [System default] Setting top_p: 0.95
+[2023-12-29 13:45:49] INFO gen_config.py:69: [System default] Setting mean_gen_len: 128
+[2023-12-29 13:45:49] INFO gen_config.py:69: [System default] Setting max_gen_len: 512
+[2023-12-29 13:45:49] INFO gen_config.py:69: [System default] Setting shift_fill_factor: 0.3
+[2023-12-29 13:45:49] INFO gen_config.py:157: Dumping configuration file to: /home/junrushao/tmp/tmpblhqsgfd/mlc-chat-config.json
+/home/junrushao/micromamba/envs/python311/bin/python -m mlc_chat convert_weight /home/junrushao/tmp/tmp5aeqhx5w/repo --quantization q4f32_1 --source-format auto --output /home/junrushao/tmp/tmpblhqsgfd
+[2023-12-29 13:45:50] INFO auto_config.py:115: Found model configuration: /home/junrushao/tmp/tmp5aeqhx5w/repo/config.json
+[2023-12-29 13:45:51] INFO auto_device.py:76: Found device: cuda:0
+[2023-12-29 13:45:51] INFO auto_device.py:76: Found device: cuda:1
+[2023-12-29 13:45:51] INFO auto_device.py:76: Found device: cuda:2
+[2023-12-29 13:45:51] INFO auto_device.py:76: Found device: cuda:3
+[2023-12-29 13:45:51] INFO auto_device.py:85: Not found device: rocm:0
+[2023-12-29 13:45:52] INFO auto_device.py:85: Not found device: metal:0
+[2023-12-29 13:45:52] INFO auto_device.py:85: Not found device: vulkan:0
+[2023-12-29 13:45:52] INFO auto_device.py:85: Not found device: opencl:0
+[2023-12-29 13:45:52] INFO auto_device.py:33: Using device: cuda:0
+[2023-12-29 13:45:52] INFO auto_weight.py:70: Finding weights in: /home/junrushao/tmp/tmp5aeqhx5w/repo
+[2023-12-29 13:45:52] INFO auto_weight.py:129: Found source weight format: huggingface-torch. Source configuration: /home/junrushao/tmp/tmp5aeqhx5w/repo/pytorch_model.bin
+[2023-12-29 13:45:52] INFO auto_weight.py:149: Not found Huggingface Safetensor
+[2023-12-29 13:45:52] INFO auto_weight.py:106: Using source weight configuration: /home/junrushao/tmp/tmp5aeqhx5w/repo/pytorch_model.bin. Use `--source` to override.
+[2023-12-29 13:45:52] INFO auto_weight.py:110: Using source weight format: huggingface-torch. Use `--source-format` to override.
+[2023-12-29 13:45:52] INFO auto_config.py:151: Found model type: gpt_bigcode. Use `--model-type` to override.
+[2023-12-29 13:45:52] INFO gpt_bigcode_model.py:41: context_window_size not found in config.json. Falling back to n_positions (8192)
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/__main__.py", line 39, in <module>
+    main()
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/__main__.py", line 28, in main
+    cli.main(sys.argv[2:])
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/cli/convert_weight.py", line 87, in main
+    convert_weight(
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/interface/convert_weight.py", line 147, in convert_weight
+    _convert_args(args)
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/interface/convert_weight.py", line 64, in _convert_args
+    model, quantize_map = args.model.quantize[args.quantization.kind](
+                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/model/gpt_bigcode/gpt_bigcode_quantization.py", line 21, in group_quant
+    model = quantization.quantize_model(
+            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/quantization/group_quantization.py", line 117, in quantize_model
+    model = mutator.visit(name_prefix, model)
+            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/tvm-dev/python/tvm/relax/frontend/nn/visitor.py", line 140, in visit
+    setattr(node, key, self.visit_module(_get_child_name(name, key), value))
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/quantization/group_quantization.py", line 113, in visit_module
+    return self.visit(name, node)
+           ^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/tvm-dev/python/tvm/relax/frontend/nn/visitor.py", line 138, in visit
+    setattr(node, key, self.visit_modulelist(_get_child_name(name, key), value))
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/tvm-dev/python/tvm/relax/frontend/nn/visitor.py", line 98, in visit_modulelist
+    return self.visit(name, node)
+           ^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/tvm-dev/python/tvm/relax/frontend/nn/visitor.py", line 130, in visit
+    node[i] = self.visit_module(f"{name}.{i}", node[i])
+              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/quantization/group_quantization.py", line 113, in visit_module
+    return self.visit(name, node)
+           ^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/tvm-dev/python/tvm/relax/frontend/nn/visitor.py", line 140, in visit
+    setattr(node, key, self.visit_module(_get_child_name(name, key), value))
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/quantization/group_quantization.py", line 113, in visit_module
+    return self.visit(name, node)
+           ^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/tvm-dev/python/tvm/relax/frontend/nn/visitor.py", line 140, in visit
+    setattr(node, key, self.visit_module(_get_child_name(name, key), value))
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/quantization/group_quantization.py", line 107, in visit_module
+    return GroupQuantizeLinear.from_linear(node, self.config)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/quantization/group_quantization.py", line 323, in from_linear
+    _apply_sharding(shard, f"{shard.name}_q_weight", quantized_linear.q_weight)
+  File "/home/junrushao/Projects/mlc-llm/python/mlc_chat/quantization/group_quantization.py", line 444, in _apply_sharding
+    assert weight.shape[0] == sum(shard.rows)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+AssertionError
+Weight conversion with arguments:
+  --config          /home/junrushao/tmp/tmp5aeqhx5w/repo/config.json
+  --quantization    GroupQuantize(name='q4f32_1', kind='group-quant', group_size=32, quantize_dtype='int4', storage_dtype='uint32', model_dtype='float32', num_elem_per_storage=8, num_storage_per_group=4, max_int_value=7)
+  --model-type      gpt_bigcode
+  --device          cuda:0
+  --source          /home/junrushao/tmp/tmp5aeqhx5w/repo/pytorch_model.bin
+  --source-format   huggingface-torch
+  --output          /home/junrushao/tmp/tmpblhqsgfd

mlc-chat-config.json

@@ -0,0 +1,40 @@
+{
+  "model_type": "gpt_bigcode",
+  "quantization": "q4f32_1",
+  "model_config": {
+    "n_embd": 6144,
+    "n_inner": 24576,
+    "n_head": 48,
+    "n_layer": 40,
+    "n_positions": 8192,
+    "layer_norm_epsilon": 1e-05,
+    "vocab_size": 49153,
+    "context_window_size": 8192,
+    "prefill_chunk_size": 8192,
+    "tensor_parallel_shards": 1
+  },
+  "vocab_size": 49153,
+  "context_window_size": 8192,
+  "sliding_window_size": -1,
+  "prefill_chunk_size": 8192,
+  "attention_sink_size": -1,
+  "tensor_parallel_shards": 1,
+  "mean_gen_len": 128,
+  "max_gen_len": 512,
+  "shift_fill_factor": 0.3,
+  "temperature": 0.7,
+  "repetition_penalty": 1.0,
+  "top_p": 0.95,
+  "conv_template": "wizard_coder_or_math",
+  "pad_token_id": 0,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "tokenizer_files": [
+    "tokenizer.json",
+    "vocab.json",
+    "merges.txt",
+    "added_tokens.json",
+    "tokenizer_config.json"
+  ],
+  "version": "0.1.0"
+}

tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<empty_output>",
+    "<commit_before>",
+    "<commit_msg>",
+    "<commit_after>",
+    "<reponame>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 2048,
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}