File size: 1,705 Bytes
c6bc767 92c7321 c6bc767 c6b1b88 c6bc767 c6b1b88 c6bc767 c6b1b88 c6bc767 c6b1b88 c6bc767 c6b1b88 c6bc767 c6b1b88 c6bc767 c6b1b88 c6bc767 c6b1b88 c6bc767 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | from torch.utils.data import Dataset, DataLoader
import torch
import io
import json
import os
import random
from datasets import load_dataset, Features, Sequence, Value
from PIL import Image
from datasets import Dataset as HFDataset
os.environ["TOKENIZERS_PARALLELISM"] = "false"
from models import VLM
def pre_processing_chat(conversations, add_system_ratio=0.2):
if any(conv.get("tools") for conv in conversations):
return conversations
SYSTEM_PROMPTS = [
"你是一个知识丰富的AI，尽力为用户提供准确的信息。",
"你是omni，一个小巧但有用的语言模型。",
"你是一个专业的AI助手，请提供有价值的回答。",
"你是omni，请尽力帮助用户解决问题。",
"你是一个可靠的AI，请给出准确的回答。",
"You are a helpful AI assistant.",
"You are omni, a lightweight intelligent assistant.",
"You are a friendly chatbot. Please answer the user's questions carefully.",
"You are a knowledgeable AI. Try your best to provide accurate information.",
"You are omni, a small but useful language model.",
]
if conversations[0].get("role") != "system":
if random.random() < add_system_ratio:
return [
{"role": "system", "content": random.choice(SYSTEM_PROMPTS)}
] + conversations
return conversations
def post_processing_chat(prompt_content, empty_think_ratio=0.2):
if (
"<think>\n\n</think>\n\n" in prompt_content
and random.random() > empty_think_ratio
):
prompt_content = prompt_content.replace("<think>\n\n</think>\n\n", "")
return prompt_content
|