Upload 5 files

app.py

@@ -0,0 +1,212 @@
+# import torch
+# from torch.utils.data import DataLoader
+# from torch.utils.tensorboard import SummaryWriter
+# from transformers import AutoModelForCausalLM, AutoTokenizer
+# from peft import LoraConfig, get_peft_model, TaskType
+# import pandas as pd
+# from qa_dataset import QADataset
+# from tqdm import tqdm
+# import os, time, sys
+#
+#
+# def train_model(model, train_loader, val_loader, optimizer, gradient_accumulation_steps,
+#                 device, num_epochs, model_output_dir, writer):
+#     batch_step = 0
+#     for epoch in range(num_epochs):
+#         time1 = time.time()
+#         model.train()
+#         for index, data in enumerate(tqdm(train_loader, file=sys.stdout, desc="Train Epoch: " + str(epoch))):
+#             input_ids = data['input_ids'].to(device, dtype=torch.long)
+#             attention_mask = data['attention_mask'].to(device, dtype=torch.long)
+#             labels = data['labels'].to(device, dtype=torch.long)
+#             # 前向传播
+#             outputs = model(
+#                 input_ids=input_ids,
+#                 attention_mask=attention_mask,
+#                 labels=labels,
+#             )
+#             loss = outputs.loss
+#             # 反向传播，计算当前梯度
+#             loss.backward()
+#             # 梯度累积步数
+#             if (index % gradient_accumulation_steps == 0 and index != 0) or index == len(train_loader) - 1:
+#                 # 更新网络参数
+#                 optimizer.step()
+#                 # 清空过往梯度
+#                 optimizer.zero_grad()
+#                 writer.add_scalar('Loss/train', loss, batch_step)
+#                 batch_step += 1
+#             # 100轮打印一次 loss
+#             if index % 100 == 0 or index == len(train_loader) - 1:
+#                 time2 = time.time()
+#                 tqdm.write(
+#                     f"{index}, epoch: {epoch} -loss: {str(loss)} ; each step's time spent: {(str(float(time2 - time1) / float(index + 0.0001)))}")
+#         # 验证
+#         model.eval()
+#         val_loss = validate_model(model, val_loader, device)
+#         writer.add_scalar('Loss/val', val_loss, epoch)
+#         print(f"val loss: {val_loss} , epoch: {epoch}")
+#         print("Save Model To ", model_output_dir)
+#         model.save_pretrained(model_output_dir)
+#
+#
+# def validate_model(model, device, val_loader):
+#     running_loss = 0.0
+#     with torch.no_grad():
+#         for _, data in enumerate(tqdm(val_loader, file=sys.stdout, desc="Validation Data")):
+#             input_ids = data['input_ids'].to(device, dtype=torch.long)
+#             attention_mask = data['attention_mask'].to(device, dtype=torch.long)
+#             labels = data['labels'].to(device, dtype=torch.long)
+#             outputs = model(
+#                 input_ids=input_ids,
+#                 attention_mask=attention_mask,
+#                 labels=labels,
+#             )
+#             loss = outputs.loss
+#             running_loss += loss.item()
+#     return running_loss / len(val_loader)
+#
+#
+# def main():
+#     # 基础模型位置
+#     model_name = "model/Qwen2-1.5B-Instruct"
+#     # 训练集
+#     train_json_path = "./data/train.json"
+#     # 验证集
+#     val_json_path = "./data/val.json"
+#     max_source_length = 128
+#     max_target_length = 256
+#     epochs = 10
+#     batch_size = 1
+#     lr = 1e-4
+#     gradient_accumulation_steps = 16
+#     lora_rank = 8
+#     lora_alpha = 32
+#     model_output_dir = "output"
+#     logs_dir = "logs"
+#     # 设备
+#     device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+#     # 加载分词器和模型
+#     tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+#     model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+#     # setup peft
+#     peft_config = LoraConfig(
+#         task_type=TaskType.CAUSAL_LM,
+#         target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+#         inference_mode=False,
+#         r=lora_rank,
+#         lora_alpha=lora_alpha,
+#         lora_dropout=0.1
+#     )
+#     model = get_peft_model(model, peft_config)
+#     model.is_parallelizable = True
+#     model.model_parallel = True
+#     model.print_trainable_parameters()
+#     print("Start Load Train Data...")
+#     train_params = {
+#         "batch_size": batch_size,
+#         "shuffle": True,
+#         "num_workers": 0,
+#     }
+#     training_set = QADataset(train_json_path, tokenizer, max_source_length, max_target_length)
+#     training_loader = DataLoader(training_set, **train_params)
+#     print("Start Load Validation Data...")
+#     val_params = {
+#         "batch_size": batch_size,
+#         "shuffle": False,
+#         "num_workers": 0,
+#     }
+#     val_set = QADataset(val_json_path, tokenizer, max_source_length, max_target_length)
+#     val_loader = DataLoader(val_set, **val_params)
+#     # 日志记录
+#     writer = SummaryWriter(logs_dir)
+#     # 优化器
+#     optimizer = torch.optim.AdamW(params=model.parameters(), lr=lr)
+#     model = model.to(device)
+#     # 开始训练
+#     print("Start Training...")
+#     train_model(
+#         model=model,
+#         train_loader=training_loader,
+#         val_loader=val_loader,
+#         optimizer=optimizer,
+#         gradient_accumulation_steps=gradient_accumulation_steps,
+#         device=device,
+#         num_epochs=epochs,
+#         model_output_dir=model_output_dir,
+#         writer=writer
+#     )
+#     writer.close()
+#
+#
+# if __name__ == '__main__':
+#     main()
+#
+from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments
+from peft import LoraConfig, get_peft_model, TaskType
+from datasets import load_dataset
+from torch.utils.tensorboard import SummaryWriter
+import os
+import torch
+def main():
+    # 基础模型位置
+    model_name = "model/Qwen2-1.5B-Instruct"
+    # 设备
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    # 加载分词器和模型
+    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+    model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+
+    # Setup PEFT (Low-Rank Adaption)
+    peft_config = LoraConfig(
+        task_type=TaskType.CAUSAL_LM,
+        target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+        inference_mode=False,
+        r=8,  # 低秩矩阵的秩
+        lora_alpha=32,  # LoRA的alpha超参数
+        lora_dropout=0.1
+    )
+    model = get_peft_model(model, peft_config)
+
+    # Load Dataset
+    train_dataset = load_dataset('json', data_files='./data/train.json', split='train')
+    val_dataset = load_dataset('json', data_files='./data/val.json', split='validation')
+
+    # Tokenize the datasets
+    def tokenize_function(examples):
+        return tokenizer(examples['input_text'], padding='max_length', truncation=True, max_length=128)
+
+    train_dataset = train_dataset.map(tokenize_function, batched=True)
+    val_dataset = val_dataset.map(tokenize_function, batched=True)
+
+    # Define Training Arguments
+    training_args = TrainingArguments(
+        output_dir="./output",             # 保存模型和日志的路径
+        evaluation_strategy="epoch",       # 每个epoch后进行验证
+        per_device_train_batch_size=1,     # 每个设备的batch size
+        per_device_eval_batch_size=1,      # 验证时的batch size
+        logging_dir="./logs",              # 日志目录
+        logging_steps=10,                  # 每10步记录一次日志
+        save_steps=100,                    # 每100步保存一次模型
+        num_train_epochs=10,               # 训练的epoch数
+        save_total_limit=2,                # 最大保存模型数
+    )
+
+    # Define the Trainer
+    trainer = Trainer(
+        model=model,                       # 训练的模型
+        args=training_args,                # 训练的参数
+        train_dataset=train_dataset,       # 训练数据集
+        eval_dataset=val_dataset,          # 验证数据集
+        tokenizer=tokenizer,               # 用于预处理的分词器
+    )
+
+    # Start Training
+    trainer.train()
+
+    # Save the model
+    model.save_pretrained('./output')
+
+if __name__ == '__main__':
+    main()

dataset.py

@@ -0,0 +1,49 @@
+import json
+import pandas as pd
+
+data_path = [
+    "./data/Chinese-medical-dialogue-data-master/Data_数据/IM_内科/内科5000-33000.csv",
+    "./data/Chinese-medical-dialogue-data-master/Data_数据/Oncology_肿瘤科/肿瘤科5-10000.csv",
+    "./data/Chinese-medical-dialogue-data-master/Data_数据/Pediatric_儿科/儿科5-14000.csv",
+    "./data/Chinese-medical-dialogue-data-master/Data_数据/Surgical_外科/外科5-14000.csv",
+]
+
+train_json_path = "./data/train.json"
+val_json_path = "./data/val.json"
+# 每个数据取 10000 条作为训练
+train_size = 10000
+# 每个数据取 2000 条作为验证
+val_size = 2000
+
+
+def main():
+    train_f = open(train_json_path, "a", encoding='utf-8')
+    val_f = open(val_json_path, "a", encoding='utf-8')
+    for path in data_path:
+        data = pd.read_csv(path, encoding='ANSI')
+        train_count = 0
+        val_count = 0
+        for index, row in data.iterrows():
+            question = row["ask"]
+            answer = row["answer"]
+            line = {
+                "question": question,
+                "answer": answer
+            }
+            line = json.dumps(line, ensure_ascii=False)
+            if train_count < train_size:
+                train_f.write(line + "\n")
+                train_count = train_count + 1
+            elif val_count < val_size:
+                val_f.write(line + "\n")
+                val_count = val_count + 1
+            else:
+                break
+    print("数据处理完毕！")
+    train_f.close()
+    val_f.close()
+
+
+if __name__ == '__main__':
+    main()
+

demo01.py

@@ -0,0 +1,2 @@
+import numpy as np
+print(np.__version__)

qa_dataset.py

@@ -0,0 +1,60 @@
+from torch.utils.data import Dataset
+import torch
+import json
+import numpy as np
+
+
+class QADataset(Dataset):
+    def __init__(self, data_path, tokenizer, max_source_length, max_target_length) -> None:
+        super().__init__()
+        self.tokenizer = tokenizer
+        self.max_source_length = max_source_length
+        self.max_target_length = max_target_length
+        self.max_seq_length = self.max_source_length + self.max_target_length
+
+        self.data = []
+        if data_path:
+            with open(data_path, "r", encoding='utf-8') as f:
+                for line in f:
+                    if not line or line == "":
+                        continue
+                    json_line = json.loads(line)
+                    question = json_line["question"]
+                    answer = json_line["answer"]
+                    self.data.append({
+                        "question": question,
+                        "answer": answer
+                    })
+        print("data load ， size：", len(self.data))
+
+    def preprocess(self, question, answer):
+        messages = [
+            {"role": "system", "content": "你是一个医疗方面的专家，可以根据患者的问题进行解答。"},
+            {"role": "user", "content": question}
+        ]
+        prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        instruction = self.tokenizer(prompt, add_special_tokens=False, max_length=self.max_source_length)
+        response = self.tokenizer(answer, add_special_tokens=False, max_length=self.max_target_length)
+        input_ids = instruction["input_ids"] + response["input_ids"] + [self.tokenizer.pad_token_id]
+        attention_mask = (instruction["attention_mask"] + response["attention_mask"] + [1])
+        labels = [-100] * len(instruction["input_ids"]) + response["input_ids"] + [self.tokenizer.pad_token_id]
+        if len(input_ids) > self.max_seq_length:
+            input_ids = input_ids[:self.max_seq_length]
+            attention_mask = attention_mask[:self.max_seq_length]
+            labels = labels[:self.max_seq_length]
+        return input_ids, attention_mask, labels
+
+    def __getitem__(self, index):
+        item_data = self.data[index]
+
+        input_ids, attention_mask, labels = self.preprocess(**item_data)
+
+        return {
+            "input_ids": torch.LongTensor(np.array(input_ids)),
+            "attention_mask": torch.LongTensor(np.array(attention_mask)),
+            "labels": torch.LongTensor(np.array(labels))
+        }
+
+    def __len__(self):
+        return len(self.data)
+

test.py

@@ -0,0 +1,34 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from peft import PeftModel
+import torch
+
+model_path = "model/Qwen2-1.5B-Instruct"
+lora_dir = "output"
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+model = AutoModelForCausalLM.from_pretrained(model_path)
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+model = PeftModel.from_pretrained(model, lora_dir)
+model.to(device)
+
+prompt = """
+5月至今上腹靠右隐痛，右背隐痛带酸，便秘，喜睡，时有腹痛，头痛，腰酸症状？
+"""
+messages = [
+    {"role": "system", "content": "你是一个医疗方面的专家，可以根据患者的问题进行解答。"},
+    {"role": "user", "content": prompt}
+]
+text = tokenizer.apply_chat_template(
+    messages,
+    tokenize=False,
+    add_generation_prompt=True
+)
+print(text)
+model_inputs = tokenizer([text], return_tensors="pt").to(device)
+generated_ids = model.generate(model_inputs.input_ids, max_new_tokens=258)
+generated_ids = [
+    output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+]
+response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+print(response)
+