# import json # import os # from transformers import AutoTokenizer # from tqdm import tqdm # # ================= 設定區 ================= # MODEL_PATH = "Salesforce/Llama-xLAM-2-8b-fc-r" # MAX_LENGTH = 16384 # INPUT_FILE = "sampled20new_fix2_fix2_toollist_remove_sys_Anyscale_no_tail.jsonl" # 輸入改成 .jsonl # OUTPUT_FILE = "sampled20new_fix2_fix2_toollist_remove_sys_Anyscale_no_tail_dropped.jsonl" # 輸出改成 .jsonl # # ========================================= # def convert_sharegpt_to_standard(conversations): # """ # (同上) 將 ShareGPT 轉換為標準格式以套用 Template # """ # new_messages = [] # for turn in conversations: # role = turn.get('from', '') # content = turn.get('value', '') # if role in ['human', 'user']: # role = 'user' # elif role in ['gpt', 'chatgpt', 'assistant', 'model']: # role = 'assistant' # elif role in ['system']: # role = 'system' # elif role in ['tool', 'function']: # role = 'tool' # new_messages.append({"role": role, "content": content}) # return new_messages # def get_accurate_token_len(tokenizer, entry): # """ # (同上) 使用 apply_chat_template 獲取真實 Token 數 # """ # messages = [] # if "conversations" in entry: # messages = convert_sharegpt_to_standard(entry["conversations"]) # elif "messages" in entry: # messages = entry["messages"] # elif "instruction" in entry: # prompt = entry.get("instruction", "") + "\n" + entry.get("input", "") # response = entry.get("output", "") # messages = [{"role": "user", "content": prompt}, {"role": "assistant", "content": response}] # if not messages: # return len(tokenizer.encode(str(entry), add_special_tokens=False)) # try: # tokenized_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False) # return len(tokenized_ids) # except Exception as e: # text = "".join([m['content'] for m in messages]) # return len(tokenizer.encode(text)) # def count_lines(filename): # """計算總行數以便顯示進度條""" # with open(filename, 'rb') as f: # return sum(1 for _ in f) # def main(): # print(f"Loading tokenizer from {MODEL_PATH}...") # try: # tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True) # except Exception as e: # print(f"Error: {e}") # return # if not os.path.exists(INPUT_FILE): # print(f"File not found: {INPUT_FILE}") # return # # 計算總行數 (非必要，但為了 tqdm 進度條好看) # print("Counting total lines...") # total_lines = count_lines(INPUT_FILE) # print(f"Processing {INPUT_FILE} ({total_lines} lines)...") # dropped_count = 0 # valid_count = 0 # # 使用 'w' 模式打開輸出檔，準備一行行寫入 # with open(INPUT_FILE, 'r', encoding='utf-8') as f_in, \ # open(OUTPUT_FILE, 'w', encoding='utf-8') as f_out: # # 逐行讀取，不佔用大量記憶體 # for line in tqdm(f_in, total=total_lines, desc="Filtering"): # line = line.strip() # if not line: continue # try: # entry = json.loads(line) # except json.JSONDecodeError: # print("Skipping invalid JSON line") # continue # length = get_accurate_token_len(tokenizer, entry) # if length <= MAX_LENGTH: # # 寫入一行 JSON 字串 # f_out.write(json.dumps(entry, ensure_ascii=False) + '\n') # valid_count += 1 # else: # dropped_count += 1 # print(f"\n===== Result =====") # print(f"Original Lines : {total_lines}") # print(f"Dropped Lines : {dropped_count} ({(dropped_count/total_lines)*100:.2f}%)") # print(f"Remaining Lines: {valid_count}") # print(f"Saved to : {OUTPUT_FILE}") # if __name__ == "__main__": # main() import json import os from transformers import AutoTokenizer from tqdm import tqdm # MODEL_PATH = "Salesforce/Llama-xLAM-2-8b-fc-r" # MAX_LENGTH = 16384 # INPUT_FILE = "sampled20new_fix2_fix2_toollist_remove_sys_Anyscale_no_tail.jsonl" # 輸入改成 .jsonl # OUTPUT_FILE = "sampled20new_fix2_fix2_toollist_remove_sys_Anyscale_no_tail_dropped.jsonl" # 輸出改成 .jsonl # ================= 設定區 ================= # MODEL_PATH = "Qwen/Qwen2.5-7B-Instruct" # MODEL_PATH = "meta-llama/Llama-3.1-8B-Instruct" MODEL_PATH = "Salesforce/Llama-xLAM-2-8b-fc-r" MAX_LENGTH = 5120 # multi_turn_miss_func_zh_tw_function_mix_sharegpt.jsonl INPUT_FILE = "multi_turn_miss_func_zh_tw_function_mix_sharegpt.jsonl" # 輸入改成 .jsonl OUTPUT_FILE = "multi_turn_miss_func_zh_tw_function_mix_sharegpt_dropped.jsonl" # 輸出改成 .jsonl DROPPED_FILE = "multi_turn_miss_func_zh_tw_function_mix_sharegpt_too_long.jsonl" # 濾掉的 (Risky) # ========================================= def convert_sharegpt_to_standard(conversations): """將 ShareGPT 轉換為標準格式以套用 Template""" new_messages = [] for turn in conversations: role = turn.get('from', '') content = turn.get('value', '') if role in ['human', 'user']: role = 'user' elif role in ['gpt', 'chatgpt', 'assistant', 'model']: role = 'assistant' elif role in ['system']: role = 'system' elif role in ['tool', 'function']: role = 'tool' new_messages.append({"role": role, "content": content}) return new_messages def get_accurate_token_len(tokenizer, entry): """使用 apply_chat_template 獲取真實 Token 數""" messages = [] if "conversations" in entry: messages = convert_sharegpt_to_standard(entry["conversations"]) elif "messages" in entry: messages = entry["messages"] elif "instruction" in entry: prompt = entry.get("instruction", "") + "\n" + entry.get("input", "") response = entry.get("output", "") messages = [{"role": "user", "content": prompt}, {"role": "assistant", "content": response}] if not messages: return len(tokenizer.encode(str(entry), add_special_tokens=False)) try: tokenized_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False) return len(tokenized_ids) except Exception: text = "".join([m['content'] for m in messages]) return len(tokenizer.encode(text)) def count_lines(filename): with open(filename, 'rb') as f: return sum(1 for _ in f) def main(): print(f"Loading tokenizer from {MODEL_PATH}...") try: tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True) except Exception as e: print(f"Error: {e}") return if not os.path.exists(INPUT_FILE): print(f"File not found: {INPUT_FILE}") return total_lines = count_lines(INPUT_FILE) print(f"Processing {INPUT_FILE} ({total_lines} lines)...") dropped_count = 0 valid_count = 0 # 同時打開三個檔案：讀取原始檔、寫入保留檔、寫入丟棄檔 with open(INPUT_FILE, 'r', encoding='utf-8') as f_in, \ open(OUTPUT_FILE, 'w', encoding='utf-8') as f_valid, \ open(DROPPED_FILE, 'w', encoding='utf-8') as f_dropped: for line in tqdm(f_in, total=total_lines, desc="Filtering"): line = line.strip() if not line: continue try: entry = json.loads(line) except json.JSONDecodeError: continue length = get_accurate_token_len(tokenizer, entry) # 準備要寫入的 JSON 字串 json_str = json.dumps(entry, ensure_ascii=False) + '\n' if length <= MAX_LENGTH: f_valid.write(json_str) valid_count += 1 else: # 這裡將過長的數據寫入 dropped file f_dropped.write(json_str) dropped_count += 1 print(f"\n===== Result =====") print(f"Original Lines : {total_lines}") print(f"Valid Lines : {valid_count} -> Saved to {OUTPUT_FILE}") print(f"Dropped Lines : {dropped_count} ({(dropped_count/total_lines)*100:.2f}%) -> Saved to {DROPPED_FILE}") if __name__ == "__main__": main()