| # import json | |
| # import os | |
| # from transformers import AutoTokenizer | |
| # from tqdm import tqdm | |
| # # ================= 設定區 ================= | |
| # MODEL_PATH = "Salesforce/Llama-xLAM-2-8b-fc-r" | |
| # MAX_LENGTH = 16384 | |
| # INPUT_FILE = "sampled20new_fix2_fix2_toollist_remove_sys_Anyscale_no_tail.jsonl" # 輸入改成 .jsonl | |
| # OUTPUT_FILE = "sampled20new_fix2_fix2_toollist_remove_sys_Anyscale_no_tail_dropped.jsonl" # 輸出改成 .jsonl | |
| # # ========================================= | |
| # def convert_sharegpt_to_standard(conversations): | |
| # """ | |
| # (同上) 將 ShareGPT 轉換為標準格式以套用 Template | |
| # """ | |
| # new_messages = [] | |
| # for turn in conversations: | |
| # role = turn.get('from', '') | |
| # content = turn.get('value', '') | |
| # if role in ['human', 'user']: | |
| # role = 'user' | |
| # elif role in ['gpt', 'chatgpt', 'assistant', 'model']: | |
| # role = 'assistant' | |
| # elif role in ['system']: | |
| # role = 'system' | |
| # elif role in ['tool', 'function']: | |
| # role = 'tool' | |
| # new_messages.append({"role": role, "content": content}) | |
| # return new_messages | |
| # def get_accurate_token_len(tokenizer, entry): | |
| # """ | |
| # (同上) 使用 apply_chat_template 獲取真實 Token 數 | |
| # """ | |
| # messages = [] | |
| # if "conversations" in entry: | |
| # messages = convert_sharegpt_to_standard(entry["conversations"]) | |
| # elif "messages" in entry: | |
| # messages = entry["messages"] | |
| # elif "instruction" in entry: | |
| # prompt = entry.get("instruction", "") + "\n" + entry.get("input", "") | |
| # response = entry.get("output", "") | |
| # messages = [{"role": "user", "content": prompt}, {"role": "assistant", "content": response}] | |
| # if not messages: | |
| # return len(tokenizer.encode(str(entry), add_special_tokens=False)) | |
| # try: | |
| # tokenized_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False) | |
| # return len(tokenized_ids) | |
| # except Exception as e: | |
| # text = "".join([m['content'] for m in messages]) | |
| # return len(tokenizer.encode(text)) | |
| # def count_lines(filename): | |
| # """計算總行數以便顯示進度條""" | |
| # with open(filename, 'rb') as f: | |
| # return sum(1 for _ in f) | |
| # def main(): | |
| # print(f"Loading tokenizer from {MODEL_PATH}...") | |
| # try: | |
| # tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True) | |
| # except Exception as e: | |
| # print(f"Error: {e}") | |
| # return | |
| # if not os.path.exists(INPUT_FILE): | |
| # print(f"File not found: {INPUT_FILE}") | |
| # return | |
| # # 計算總行數 (非必要，但為了 tqdm 進度條好看) | |
| # print("Counting total lines...") | |
| # total_lines = count_lines(INPUT_FILE) | |
| # print(f"Processing {INPUT_FILE} ({total_lines} lines)...") | |
| # dropped_count = 0 | |
| # valid_count = 0 | |
| # # 使用 'w' 模式打開輸出檔，準備一行行寫入 | |
| # with open(INPUT_FILE, 'r', encoding='utf-8') as f_in, \ | |
| # open(OUTPUT_FILE, 'w', encoding='utf-8') as f_out: | |
| # # 逐行讀取，不佔用大量記憶體 | |
| # for line in tqdm(f_in, total=total_lines, desc="Filtering"): | |
| # line = line.strip() | |
| # if not line: continue | |
| # try: | |
| # entry = json.loads(line) | |
| # except json.JSONDecodeError: | |
| # print("Skipping invalid JSON line") | |
| # continue | |
| # length = get_accurate_token_len(tokenizer, entry) | |
| # if length <= MAX_LENGTH: | |
| # # 寫入一行 JSON 字串 | |
| # f_out.write(json.dumps(entry, ensure_ascii=False) + '\n') | |
| # valid_count += 1 | |
| # else: | |
| # dropped_count += 1 | |
| # print(f"\n===== Result =====") | |
| # print(f"Original Lines : {total_lines}") | |
| # print(f"Dropped Lines : {dropped_count} ({(dropped_count/total_lines)*100:.2f}%)") | |
| # print(f"Remaining Lines: {valid_count}") | |
| # print(f"Saved to : {OUTPUT_FILE}") | |
| # if __name__ == "__main__": | |
| # main() | |
| import json | |
| import os | |
| from transformers import AutoTokenizer | |
| from tqdm import tqdm | |
| # MODEL_PATH = "Salesforce/Llama-xLAM-2-8b-fc-r" | |
| # MAX_LENGTH = 16384 | |
| # INPUT_FILE = "sampled20new_fix2_fix2_toollist_remove_sys_Anyscale_no_tail.jsonl" # 輸入改成 .jsonl | |
| # OUTPUT_FILE = "sampled20new_fix2_fix2_toollist_remove_sys_Anyscale_no_tail_dropped.jsonl" # 輸出改成 .jsonl | |
| # ================= 設定區 ================= | |
| # MODEL_PATH = "Qwen/Qwen2.5-7B-Instruct" | |
| # MODEL_PATH = "meta-llama/Llama-3.1-8B-Instruct" | |
| MODEL_PATH = "Salesforce/Llama-xLAM-2-8b-fc-r" | |
| MAX_LENGTH = 5120 | |
| # multi_turn_miss_func_zh_tw_function_mix_sharegpt.jsonl | |
| INPUT_FILE = "multi_turn_miss_func_zh_tw_function_mix_sharegpt.jsonl" # 輸入改成 .jsonl | |
| OUTPUT_FILE = "multi_turn_miss_func_zh_tw_function_mix_sharegpt_dropped.jsonl" # 輸出改成 .jsonl | |
| DROPPED_FILE = "multi_turn_miss_func_zh_tw_function_mix_sharegpt_too_long.jsonl" # 濾掉的 (Risky) | |
| # ========================================= | |
| def convert_sharegpt_to_standard(conversations): | |
| """將 ShareGPT 轉換為標準格式以套用 Template""" | |
| new_messages = [] | |
| for turn in conversations: | |
| role = turn.get('from', '') | |
| content = turn.get('value', '') | |
| if role in ['human', 'user']: role = 'user' | |
| elif role in ['gpt', 'chatgpt', 'assistant', 'model']: role = 'assistant' | |
| elif role in ['system']: role = 'system' | |
| elif role in ['tool', 'function']: role = 'tool' | |
| new_messages.append({"role": role, "content": content}) | |
| return new_messages | |
| def get_accurate_token_len(tokenizer, entry): | |
| """使用 apply_chat_template 獲取真實 Token 數""" | |
| messages = [] | |
| if "conversations" in entry: | |
| messages = convert_sharegpt_to_standard(entry["conversations"]) | |
| elif "messages" in entry: | |
| messages = entry["messages"] | |
| elif "instruction" in entry: | |
| prompt = entry.get("instruction", "") + "\n" + entry.get("input", "") | |
| response = entry.get("output", "") | |
| messages = [{"role": "user", "content": prompt}, {"role": "assistant", "content": response}] | |
| if not messages: | |
| return len(tokenizer.encode(str(entry), add_special_tokens=False)) | |
| try: | |
| tokenized_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False) | |
| return len(tokenized_ids) | |
| except Exception: | |
| text = "".join([m['content'] for m in messages]) | |
| return len(tokenizer.encode(text)) | |
| def count_lines(filename): | |
| with open(filename, 'rb') as f: | |
| return sum(1 for _ in f) | |
| def main(): | |
| print(f"Loading tokenizer from {MODEL_PATH}...") | |
| try: | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True) | |
| except Exception as e: | |
| print(f"Error: {e}") | |
| return | |
| if not os.path.exists(INPUT_FILE): | |
| print(f"File not found: {INPUT_FILE}") | |
| return | |
| total_lines = count_lines(INPUT_FILE) | |
| print(f"Processing {INPUT_FILE} ({total_lines} lines)...") | |
| dropped_count = 0 | |
| valid_count = 0 | |
| # 同時打開三個檔案：讀取原始檔、寫入保留檔、寫入丟棄檔 | |
| with open(INPUT_FILE, 'r', encoding='utf-8') as f_in, \ | |
| open(OUTPUT_FILE, 'w', encoding='utf-8') as f_valid, \ | |
| open(DROPPED_FILE, 'w', encoding='utf-8') as f_dropped: | |
| for line in tqdm(f_in, total=total_lines, desc="Filtering"): | |
| line = line.strip() | |
| if not line: continue | |
| try: | |
| entry = json.loads(line) | |
| except json.JSONDecodeError: | |
| continue | |
| length = get_accurate_token_len(tokenizer, entry) | |
| # 準備要寫入的 JSON 字串 | |
| json_str = json.dumps(entry, ensure_ascii=False) + '\n' | |
| if length <= MAX_LENGTH: | |
| f_valid.write(json_str) | |
| valid_count += 1 | |
| else: | |
| # 這裡將過長的數據寫入 dropped file | |
| f_dropped.write(json_str) | |
| dropped_count += 1 | |
| print(f"\n===== Result =====") | |
| print(f"Original Lines : {total_lines}") | |
| print(f"Valid Lines : {valid_count} -> Saved to {OUTPUT_FILE}") | |
| print(f"Dropped Lines : {dropped_count} ({(dropped_count/total_lines)*100:.2f}%) -> Saved to {DROPPED_FILE}") | |
| if __name__ == "__main__": | |
| main() |