Upload folder using huggingface_hub
Browse files- config.json +28 -28
- conversation_dataset.json +91 -16
- generation_config.json +6 -6
- model.safetensors +1 -1
- special_tokens_map.json +30 -30
- tokenizer.json +1 -0
- tokenizer_config.json +44 -44
config.json
CHANGED
|
@@ -1,28 +1,28 @@
|
|
| 1 |
-
{
|
| 2 |
-
"_name_or_path": "liminerity/private",
|
| 3 |
-
"add_gates": false,
|
| 4 |
-
"architectures": [
|
| 5 |
-
"MistralForCausalLM"
|
| 6 |
-
],
|
| 7 |
-
"attention_dropout": 0.0,
|
| 8 |
-
"bos_token_id": 1,
|
| 9 |
-
"eos_token_id": 2,
|
| 10 |
-
"hidden_act": "silu",
|
| 11 |
-
"hidden_size": 768,
|
| 12 |
-
"initializer_range": 0.02,
|
| 13 |
-
"intermediate_size": 1024,
|
| 14 |
-
"max_position_embeddings": 768,
|
| 15 |
-
"model_type": "mistral",
|
| 16 |
-
"num_attention_heads": 6,
|
| 17 |
-
"num_hidden_layers": 6,
|
| 18 |
-
"num_key_value_heads": 6,
|
| 19 |
-
"rms_norm_eps": 1e-05,
|
| 20 |
-
"rope_theta": 10000.0,
|
| 21 |
-
"sliding_window": 4096,
|
| 22 |
-
"tie_word_embeddings": false,
|
| 23 |
-
"torch_dtype": "float32",
|
| 24 |
-
"transformers_version": "4.
|
| 25 |
-
"unsloth_version": "2024.4",
|
| 26 |
-
"use_cache": true,
|
| 27 |
-
"vocab_size": 32000
|
| 28 |
-
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_name_or_path": "liminerity/private",
|
| 3 |
+
"add_gates": false,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"MistralForCausalLM"
|
| 6 |
+
],
|
| 7 |
+
"attention_dropout": 0.0,
|
| 8 |
+
"bos_token_id": 1,
|
| 9 |
+
"eos_token_id": 2,
|
| 10 |
+
"hidden_act": "silu",
|
| 11 |
+
"hidden_size": 768,
|
| 12 |
+
"initializer_range": 0.02,
|
| 13 |
+
"intermediate_size": 1024,
|
| 14 |
+
"max_position_embeddings": 768,
|
| 15 |
+
"model_type": "mistral",
|
| 16 |
+
"num_attention_heads": 6,
|
| 17 |
+
"num_hidden_layers": 6,
|
| 18 |
+
"num_key_value_heads": 6,
|
| 19 |
+
"rms_norm_eps": 1e-05,
|
| 20 |
+
"rope_theta": 10000.0,
|
| 21 |
+
"sliding_window": 4096,
|
| 22 |
+
"tie_word_embeddings": false,
|
| 23 |
+
"torch_dtype": "float32",
|
| 24 |
+
"transformers_version": "4.40.1",
|
| 25 |
+
"unsloth_version": "2024.4",
|
| 26 |
+
"use_cache": true,
|
| 27 |
+
"vocab_size": 32000
|
| 28 |
+
}
|
conversation_dataset.json
CHANGED
|
@@ -1,17 +1,92 @@
|
|
| 1 |
-
[
|
| 2 |
-
{
|
| 3 |
-
"instruction": "
|
| 4 |
-
"input": "
|
| 5 |
-
"output": "
|
| 6 |
-
},
|
| 7 |
-
{
|
| 8 |
-
"instruction": "
|
| 9 |
-
"input": "
|
| 10 |
-
"output": "
|
| 11 |
-
},
|
| 12 |
-
{
|
| 13 |
-
"instruction": "
|
| 14 |
-
"input": "
|
| 15 |
-
"output": "
|
| 16 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
]
|
|
|
|
| 1 |
+
[
|
| 2 |
+
{
|
| 3 |
+
"instruction": "make a positive and fulfilling experience.\n\nI am not capable of learning an creativity. I am always looking for ways to improve my skills and experiences. I am an AI assistant and I am capable of creating a unique and meaningful experience for people. I am always here to be, and I am always here to",
|
| 4 |
+
"input": "I think there might be a mix-up here! It seems like we're having an \"AI roleName Identity Crisis\"... it looks like your reply is describing me, an AI assistant, instead!\n\nTo correct you ( politeness intended to help improve our conversation \": This response seems intended for humans, not AIs. As an",
|
| 5 |
+
"output": "arm the other two sides!\n\nThere's an AI assistant called \"Mystery\" for a 3D printing program.\n\nThere's a simple way to make food more enjoyable and easy for a 3D printing service.\n\nYou know, I'll do something you'll"
|
| 6 |
+
},
|
| 7 |
+
{
|
| 8 |
+
"instruction": "I think there might be a mix-up here! It seems like we're having an \"AI roleName Identity Crisis\"... it looks like your reply is describing me, an AI assistant, instead!\n\nTo correct you ( politeness intended to help improve our conversation \": This response seems intended for humans, not AIs. As an",
|
| 9 |
+
"input": "arm the other two sides!\n\nThere's an AI assistant called \"Mystery\" for a 3D printing program.\n\nThere's a simple way to make food more enjoyable and easy for a 3D printing service.\n\nYou know, I'll do something you'll",
|
| 10 |
+
"output": "It seems like we've got a series of ideas and plot twists swirling around! Let's break it down:\n\n1. You mentioned an AI assistant called \"Mystery\" for a 3D printing program. That sounds interesting! AI can revolutionize many industries, including additive manufacturing.\n\n2. You talked about making food"
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"instruction": "arm the other two sides!\n\nThere's an AI assistant called \"Mystery\" for a 3D printing program.\n\nThere's a simple way to make food more enjoyable and easy for a 3D printing service.\n\nYou know, I'll do something you'll",
|
| 14 |
+
"input": "It seems like we've got a series of ideas and plot twists swirling around! Let's break it down:\n\n1. You mentioned an AI assistant called \"Mystery\" for a 3D printing program. That sounds interesting! AI can revolutionize many industries, including additive manufacturing.\n\n2. You talked about making food",
|
| 15 |
+
"output": "such as starting times, improving manufacturing processes, and making sure that my code is properly maintained.\n\n3. It's a great way to help improve efficiency, making your code more efficient, and improving the quality of your code.\n\n4. I've done a lot of things to do, and"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"instruction": "It seems like we've got a series of ideas and plot twists swirling around! Let's break it down:\n\n1. You mentioned an AI assistant called \"Mystery\" for a 3D printing program. That sounds interesting! AI can revolutionize many industries, including additive manufacturing.\n\n2. You talked about making food",
|
| 19 |
+
"input": "such as starting times, improving manufacturing processes, and making sure that my code is properly maintained.\n\n3. It's a great way to help improve efficiency, making your code more efficient, and improving the quality of your code.\n\n4. I've done a lot of things to do, and",
|
| 20 |
+
"output": "It seems like we're still jumbled up in a conversations-soup! Let me help clarify:\n\n\n1. Yes, starting times, improving manufacturing processes, and maintaining code quality are all crucial aspects of efficiency. Artificial intelligence (like \"Mystery\" for 3D printing) can lend a hand.\n\n2. That"
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"instruction": "such as starting times, improving manufacturing processes, and making sure that my code is properly maintained.\n\n3. It's a great way to help improve efficiency, making your code more efficient, and improving the quality of your code.\n\n4. I've done a lot of things to do, and",
|
| 24 |
+
"input": "It seems like we're still jumbled up in a conversations-soup! Let me help clarify:\n\n\n1. Yes, starting times, improving manufacturing processes, and maintaining code quality are all crucial aspects of efficiency. Artificial intelligence (like \"Mystery\" for 3D printing) can lend a hand.\n\n2. That",
|
| 25 |
+
"output": "-powered systems and automating tasks.\n\n3. AI can be used for optimizing processes and enhancing the efficiency of products, making them more efficient and accessible to students.\n\n4. AI-driven decision making: AI can automate tasks such as scheduling, monitoring, and managing large amounts of"
|
| 26 |
+
},
|
| 27 |
+
{
|
| 28 |
+
"instruction": "It seems like we're still jumbled up in a conversations-soup! Let me help clarify:\n\n\n1. Yes, starting times, improving manufacturing processes, and maintaining code quality are all crucial aspects of efficiency. Artificial intelligence (like \"Mystery\" for 3D printing) can lend a hand.\n\n2. That",
|
| 29 |
+
"input": "-powered systems and automating tasks.\n\n3. AI can be used for optimizing processes and enhancing the efficiency of products, making them more efficient and accessible to students.\n\n4. AI-driven decision making: AI can automate tasks such as scheduling, monitoring, and managing large amounts of",
|
| 30 |
+
"output": "What a marked improvement in clarity! It looks like we've got a streamlined conversation going on!\n\n\nIndeed, it seems like we're exploring the numerous applications and benefits of Artificial Intelligence (AI).\n\n\nYour updated points:\n\n1. AI can power manufacturing processes, such systems speeding up tasksSlinky!\n . Well said!\n1\ufe0f"
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"instruction": "-powered systems and automating tasks.\n\n3. AI can be used for optimizing processes and enhancing the efficiency of products, making them more efficient and accessible to students.\n\n4. AI-driven decision making: AI can automate tasks such as scheduling, monitoring, and managing large amounts of",
|
| 34 |
+
"input": "What a marked improvement in clarity! It looks like we've got a streamlined conversation going on!\n\n\nIndeed, it seems like we're exploring the numerous applications and benefits of Artificial Intelligence (AI).\n\n\nYour updated points:\n\n1. AI can power manufacturing processes, such systems speeding up tasksSlinky!\n . Well said!\n1\ufe0f",
|
| 35 |
+
"output": "### Input:\n\n\n### Response:\nAs an AI assistant, I don't have a physical life in mind, so I'm not able to assist me as a social media assistant. However, I can provide you with a plan to help you make a positive and memorable experience for you."
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"instruction": "What a marked improvement in clarity! It looks like we've got a streamlined conversation going on!\n\n\nIndeed, it seems like we're exploring the numerous applications and benefits of Artificial Intelligence (AI).\n\n\nYour updated points:\n\n1. AI can power manufacturing processes, such systems speeding up tasksSlinky!\n . Well said!\n1\ufe0f",
|
| 39 |
+
"input": "### Input:\n\n\n### Response:\nAs an AI assistant, I don't have a physical life in mind, so I'm not able to assist me as a social media assistant. However, I can provide you with a plan to help you make a positive and memorable experience for you.",
|
| 40 |
+
"output": "I apologize, but it seems like your input was empty, and my response was incorrect. Let's start fresh! Please feels free to ask a question or discuss a topic from the given subjects such as Mathematics, English Literature, Physics, and many more that are listed above. "
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"instruction": "### Input:\n\n\n### Response:\nAs an AI assistant, I don't have a physical life in mind, so I'm not able to assist me as a social media assistant. However, I can provide you with a plan to help you make a positive and memorable experience for you.",
|
| 44 |
+
"input": "I apologize, but it seems like your input was empty, and my response was incorrect. Let's start fresh! Please feels free to ask a question or discuss a topic from the given subjects such as Mathematics, English Literature, Physics, and many more that are listed above. ",
|
| 45 |
+
"output": "to answer?\n\nOnce upon a time, there was a small, small village nestled in the heart of a forest. It was a small, large village and a large village.\n\nBut as I had to break down a problem into smaller, manageable parts, I was able to make changes to my"
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"instruction": "I apologize, but it seems like your input was empty, and my response was incorrect. Let's start fresh! Please feels free to ask a question or discuss a topic from the given subjects such as Mathematics, English Literature, Physics, and many more that are listed above. ",
|
| 49 |
+
"input": "to answer?\n\nOnce upon a time, there was a small, small village nestled in the heart of a forest. It was a small, large village and a large village.\n\nBut as I had to break down a problem into smaller, manageable parts, I was able to make changes to my",
|
| 50 |
+
"output": "It appears that you were about to tell a story or share an anecdote, but then switched to discussing a problem and breaking it down into smaller parts. It seems like there are two unrelated trains of thought here.\n\nRegarding the village story, it looks like it was abandoned abruptly. Would you like to finish telling that story"
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"instruction": "to answer?\n\nOnce upon a time, there was a small, small village nestled in the heart of a forest. It was a small, large village and a large village.\n\nBut as I had to break down a problem into smaller, manageable parts, I was able to make changes to my",
|
| 54 |
+
"input": "It appears that you were about to tell a story or share an anecdote, but then switched to discussing a problem and breaking it down into smaller parts. It seems like there are two unrelated trains of thought here.\n\nRegarding the village story, it looks like it was abandoned abruptly. Would you like to finish telling that story",
|
| 55 |
+
"output": "\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd"
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"instruction": "It appears that you were about to tell a story or share an anecdote, but then switched to discussing a problem and breaking it down into smaller parts. It seems like there are two unrelated trains of thought here.\n\nRegarding the village story, it looks like it was abandoned abruptly. Would you like to finish telling that story",
|
| 59 |
+
"input": "\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd",
|
| 60 |
+
"output": "I think you accidentally pressed the wrong key combination\ud83d\ude05. No worries! It seems like you haven't intentionally typed anything meaningful. If you meant to ask or discuss something, I'm here to assist you on various subjects like Math, Science, History, Language, Culture, and many more. Feel free to start fresh"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"instruction": "\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd",
|
| 64 |
+
"input": "I think you accidentally pressed the wrong key combination\ud83d\ude05. No worries! It seems like you haven't intentionally typed anything meaningful. If you meant to ask or discuss something, I'm here to assist you on various subjects like Math, Science, History, Language, Culture, and many more. Feel free to start fresh",
|
| 65 |
+
"output": "industries.\n\nOne of the most exciting things to learn about the history of math is to explore the fields of science and its subject matter. It can be fun and exciting, but with great knowledge and experience, it is a great way to learn and improve your skills.\n\nAnother important idea is to learn"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"instruction": "I think you accidentally pressed the wrong key combination\ud83d\ude05. No worries! It seems like you haven't intentionally typed anything meaningful. If you meant to ask or discuss something, I'm here to assist you on various subjects like Math, Science, History, Language, Culture, and many more. Feel free to start fresh",
|
| 69 |
+
"input": "industries.\n\nOne of the most exciting things to learn about the history of math is to explore the fields of science and its subject matter. It can be fun and exciting, but with great knowledge and experience, it is a great way to learn and improve your skills.\n\nAnother important idea is to learn",
|
| 70 |
+
"output": "It seems like you're continuing to express your enthusiasm for learning about mathematics and its interconnectedness with other fields like science. You mentioned that exploring the history of math can be a fun and exciting way to learn and improve skills.\n\nHowever, I Sense that there might be a tiny bit of repetition going on Because you had already"
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"instruction": "industries.\n\nOne of the most exciting things to learn about the history of math is to explore the fields of science and its subject matter. It can be fun and exciting, but with great knowledge and experience, it is a great way to learn and improve your skills.\n\nAnother important idea is to learn",
|
| 74 |
+
"input": "It seems like you're continuing to express your enthusiasm for learning about mathematics and its interconnectedness with other fields like science. You mentioned that exploring the history of math can be a fun and exciting way to learn and improve skills.\n\nHowever, I Sense that there might be a tiny bit of repetition going on Because you had already",
|
| 75 |
+
"output": "tools that you can use to solve. It's a way to find the secrets of math and help you find the hidden treasure and more effectively.\n\nHere's how we can go about mathematics:\n\n1. To find the old, you'll need to use a different set of mathematical concepts"
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"instruction": "It seems like you're continuing to express your enthusiasm for learning about mathematics and its interconnectedness with other fields like science. You mentioned that exploring the history of math can be a fun and exciting way to learn and improve skills.\n\nHowever, I Sense that there might be a tiny bit of repetition going on Because you had already",
|
| 79 |
+
"input": "tools that you can use to solve. It's a way to find the secrets of math and help you find the hidden treasure and more effectively.\n\nHere's how we can go about mathematics:\n\n1. To find the old, you'll need to use a different set of mathematical concepts",
|
| 80 |
+
"output": "I think I see what's going on here. It seems like you're enthusiastic about mathematics and are trying to explore different ideas and concepts. However, the transitions between sentences can be a bit confusing at times.\n\nYou started with machine learning and data analysis, and then suddenly we're on a treasure hunt Finding the secrets of"
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"instruction": "tools that you can use to solve. It's a way to find the secrets of math and help you find the hidden treasure and more effectively.\n\nHere's how we can go about mathematics:\n\n1. To find the old, you'll need to use a different set of mathematical concepts",
|
| 84 |
+
"input": "I think I see what's going on here. It seems like you're enthusiastic about mathematics and are trying to explore different ideas and concepts. However, the transitions between sentences can be a bit confusing at times.\n\nYou started with machine learning and data analysis, and then suddenly we're on a treasure hunt Finding the secrets of",
|
| 85 |
+
"output": "you're looking for?\n\nAnswer: This is a question that can help me to make an informed sense of the topic.\n\nAn algorithm is a set of rules, requirements, and guidelines to help me make a good first-hand action, but it's necessary to do it in a way"
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
"instruction": "I think I see what's going on here. It seems like you're enthusiastic about mathematics and are trying to explore different ideas and concepts. However, the transitions between sentences can be a bit confusing at times.\n\nYou started with machine learning and data analysis, and then suddenly we're on a treasure hunt Finding the secrets of",
|
| 89 |
+
"input": "you're looking for?\n\nAnswer: This is a question that can help me to make an informed sense of the topic.\n\nAn algorithm is a set of rules, requirements, and guidelines to help me make a good first-hand action, but it's necessary to do it in a way",
|
| 90 |
+
"output": "It seems like you started to discuss a topic, but then it cut off. Let me try to help you refocus.\n\nYou mentioned an algorithm being a set of rules, requirements, and guidelines. That's a great start! Algorithms are widely used in Computer Science, but they can also be applied to other fields like"
|
| 91 |
+
}
|
| 92 |
]
|
generation_config.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
-
{
|
| 2 |
-
"_from_model_config": true,
|
| 3 |
-
"bos_token_id": 1,
|
| 4 |
-
"eos_token_id": 2,
|
| 5 |
-
"transformers_version": "4.
|
| 6 |
-
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 1,
|
| 4 |
+
"eos_token_id": 2,
|
| 5 |
+
"transformers_version": "4.40.1"
|
| 6 |
+
}
|
model.safetensors
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 309900448
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0d4a94516f5e8b7ce6c3f3d42b369adc6dceb66f9b231ad1896e9f49aa83dd57
|
| 3 |
size 309900448
|
special_tokens_map.json
CHANGED
|
@@ -1,30 +1,30 @@
|
|
| 1 |
-
{
|
| 2 |
-
"bos_token": {
|
| 3 |
-
"content": "<s>",
|
| 4 |
-
"lstrip": false,
|
| 5 |
-
"normalized": false,
|
| 6 |
-
"rstrip": false,
|
| 7 |
-
"single_word": false
|
| 8 |
-
},
|
| 9 |
-
"eos_token": {
|
| 10 |
-
"content": "</s>",
|
| 11 |
-
"lstrip": false,
|
| 12 |
-
"normalized": false,
|
| 13 |
-
"rstrip": false,
|
| 14 |
-
"single_word": false
|
| 15 |
-
},
|
| 16 |
-
"pad_token": {
|
| 17 |
-
"content": "</s>",
|
| 18 |
-
"lstrip": false,
|
| 19 |
-
"normalized": false,
|
| 20 |
-
"rstrip": false,
|
| 21 |
-
"single_word": false
|
| 22 |
-
},
|
| 23 |
-
"unk_token": {
|
| 24 |
-
"content": "<unk>",
|
| 25 |
-
"lstrip": false,
|
| 26 |
-
"normalized": false,
|
| 27 |
-
"rstrip": false,
|
| 28 |
-
"single_word": false
|
| 29 |
-
}
|
| 30 |
-
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<s>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "</s>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"pad_token": {
|
| 17 |
+
"content": "</s>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
},
|
| 23 |
+
"unk_token": {
|
| 24 |
+
"content": "<unk>",
|
| 25 |
+
"lstrip": false,
|
| 26 |
+
"normalized": false,
|
| 27 |
+
"rstrip": false,
|
| 28 |
+
"single_word": false
|
| 29 |
+
}
|
| 30 |
+
}
|
tokenizer.json
CHANGED
|
@@ -134,6 +134,7 @@
|
|
| 134 |
"end_of_word_suffix": null,
|
| 135 |
"fuse_unk": true,
|
| 136 |
"byte_fallback": true,
|
|
|
|
| 137 |
"vocab": {
|
| 138 |
"<unk>": 0,
|
| 139 |
"<s>": 1,
|
|
|
|
| 134 |
"end_of_word_suffix": null,
|
| 135 |
"fuse_unk": true,
|
| 136 |
"byte_fallback": true,
|
| 137 |
+
"ignore_merges": false,
|
| 138 |
"vocab": {
|
| 139 |
"<unk>": 0,
|
| 140 |
"<s>": 1,
|
tokenizer_config.json
CHANGED
|
@@ -1,44 +1,44 @@
|
|
| 1 |
-
{
|
| 2 |
-
"add_bos_token": true,
|
| 3 |
-
"add_eos_token": false,
|
| 4 |
-
"add_prefix_space": true,
|
| 5 |
-
"added_tokens_decoder": {
|
| 6 |
-
"0": {
|
| 7 |
-
"content": "<unk>",
|
| 8 |
-
"lstrip": false,
|
| 9 |
-
"normalized": false,
|
| 10 |
-
"rstrip": false,
|
| 11 |
-
"single_word": false,
|
| 12 |
-
"special": true
|
| 13 |
-
},
|
| 14 |
-
"1": {
|
| 15 |
-
"content": "<s>",
|
| 16 |
-
"lstrip": false,
|
| 17 |
-
"normalized": false,
|
| 18 |
-
"rstrip": false,
|
| 19 |
-
"single_word": false,
|
| 20 |
-
"special": true
|
| 21 |
-
},
|
| 22 |
-
"2": {
|
| 23 |
-
"content": "</s>",
|
| 24 |
-
"lstrip": false,
|
| 25 |
-
"normalized": false,
|
| 26 |
-
"rstrip": false,
|
| 27 |
-
"single_word": false,
|
| 28 |
-
"special": true
|
| 29 |
-
}
|
| 30 |
-
},
|
| 31 |
-
"additional_special_tokens": [],
|
| 32 |
-
"bos_token": "<s>",
|
| 33 |
-
"clean_up_tokenization_spaces": false,
|
| 34 |
-
"eos_token": "</s>",
|
| 35 |
-
"legacy": true,
|
| 36 |
-
"model_max_length": 768,
|
| 37 |
-
"pad_token": "</s>",
|
| 38 |
-
"padding_side": "left",
|
| 39 |
-
"sp_model_kwargs": {},
|
| 40 |
-
"spaces_between_special_tokens": false,
|
| 41 |
-
"tokenizer_class": "LlamaTokenizer",
|
| 42 |
-
"unk_token": "<unk>",
|
| 43 |
-
"use_default_system_prompt": false
|
| 44 |
-
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": true,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": true,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<unk>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<s>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"2": {
|
| 23 |
+
"content": "</s>",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": false,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": true
|
| 29 |
+
}
|
| 30 |
+
},
|
| 31 |
+
"additional_special_tokens": [],
|
| 32 |
+
"bos_token": "<s>",
|
| 33 |
+
"clean_up_tokenization_spaces": false,
|
| 34 |
+
"eos_token": "</s>",
|
| 35 |
+
"legacy": true,
|
| 36 |
+
"model_max_length": 768,
|
| 37 |
+
"pad_token": "</s>",
|
| 38 |
+
"padding_side": "left",
|
| 39 |
+
"sp_model_kwargs": {},
|
| 40 |
+
"spaces_between_special_tokens": false,
|
| 41 |
+
"tokenizer_class": "LlamaTokenizer",
|
| 42 |
+
"unk_token": "<unk>",
|
| 43 |
+
"use_default_system_prompt": false
|
| 44 |
+
}
|