update

examples/tutorials/lora_unsloth/step_2_train_model.py

@@ -103,6 +103,7 @@ def main():
     train_dataset = train_dataset.map(
         format_func,
         batched=False,
+        remove_columns=train_dataset.column_names,
     )
     print(train_dataset)
 

main.py

@@ -1,16 +1,88 @@
-# 这是一个示例 Python 脚本。
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import asyncio
+import logging
+from pathlib import Path
+import platform
 
-# 按 ⌃R 执行或将其替换为您的代码。
-# 按 双击 ⇧ 在所有地方搜索类、文件、工具窗口、操作和设置。
+import gradio as gr
 
+import log
+from project_settings import environment, project_path, log_directory, time_zone_info
 
-def print_hi(name):
-    # 在下面的代码行中使用断点来调试脚本。
-    print(f'Hi, {name}')  # 按 ⌘F8 切换断点。
+log.setup_size_rotating(log_directory=log_directory, tz_info=time_zone_info)
 
+from tabs.chat_template_tab import get_chat_template_tab
+from tabs.shell_tab import get_shell_tab
 
-# 按装订区域中的绿色按钮以运行脚本。
-if __name__ == '__main__':
-    print_hi('PyCharm')
 
-# 访问 https://www.jetbrains.com/help/pycharm/ 获取 PyCharm 帮助
+logger = logging.getLogger("main")
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--porter_tasks_file_dir",
+        default=(project_path / "data/porter_tasks").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--live_recorder_tasks_file",
+        default=(project_path / "data/live_recorder_tasks.json").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--video_download_tasks_file",
+        default=(project_path / "data/video_download_tasks.json").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--youtube_video_upload_tasks_file",
+        default=(project_path / "data/youtube_video_upload_tasks.json").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--bilibili_video_upload_tasks_file",
+        default=(project_path / "data/bilibili_video_upload_tasks.json").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--live_records_dir",
+        default=(project_path / "data/live_records").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--server_port",
+        default=environment.get("server_port", 7860),
+        type=int
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+
+def main():
+    args = get_args()
+
+    # ui
+    with gr.Blocks() as blocks:
+        gr.Markdown(value="live recording.")
+        with gr.Tabs():
+            _ = get_chat_template_tab()
+            _ = get_shell_tab()
+
+    # http://127.0.0.1:7870/
+    # http://10.75.27.247:7870/
+    blocks.queue().launch(
+        # share=True,
+        share=False if platform.system() in ("Windows", "Darwin") else False,
+        server_name="127.0.0.1" if platform.system() in ("Windows", "Darwin") else "0.0.0.0",
+        server_port=args.server_port
+    )
+    return
+
+
+if __name__ == "__main__":
+    main()

project_settings.py

@@ -9,6 +9,8 @@ from toolbox.os.environment import EnvironmentManager
 project_path = os.path.abspath(os.path.dirname(__file__))
 project_path = Path(project_path)
 
+time_zone_info = "Asia/Shanghai"
+
 log_directory = project_path / "logs"
 log_directory.mkdir(parents=True, exist_ok=True)
 

requirements.txt

@@ -0,0 +1 @@
+gradio

tabs/chat_template_tab.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import json
+
+import gradio as gr
+from transformers import AutoTokenizer
+
+
+def run_chat_template(conversation: str, model_name: str, add_generation_prompt: bool = False):
+    conversation = json.loads(conversation)
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+    result = tokenizer.apply_chat_template(
+        conversation,
+        tokenize=False,
+        add_generation_prompt=add_generation_prompt,
+    )
+    return result
+
+
+def get_chat_template_tab():
+    with gr.TabItem("chat_template"):
+        model_name_choices = ["unsloth/Qwen3-8B-unsloth-bnb-4bit"]
+        ct_model_name = gr.Dropdown(choices=model_name_choices, value=model_name_choices[0], label="model_name")
+        ct_conversation = gr.Textbox(label="conversation")
+        ct_add_generation_prompt = gr.Checkbox(label="add_generation_prompt")
+        ct_tokenize = gr.Button("tokenize")
+        ct_output = gr.Textbox(label="output", max_lines=100)
+
+        ct_tokenize.click(
+            run_chat_template,
+            inputs=[ct_conversation, ct_model_name, ct_add_generation_prompt],
+            outputs=[ct_output],
+        )
+
+        gr.Examples(
+            examples=[
+                [
+                    json.dumps([{"role": "user", "content": "帮我识别出文本中的关键词：\n凉山彝族社会中的\"尔普\"(份子钱)是一种礼物交换形式.对\"尔普\"的研究和分析,可有助于人们理解凉山彝族社会.\"尔普\"本来是维系彝族传统社会宗族内部亲属组织的纽带,由于文化变迁的原因,后来发展出了跨宗族的\"尔普\"新形式,又由于族群互动的原因,还产生了跨越族群的\"尔普\"形式.\"尔普\"形式的变迁是族群互动下的一种文化变迁形式,其动力来源于彝、汉两族的互动关系.彝族社会中\"尔普\"的变迁形式是人类学关于族群互动下的文化变迁理论的鲜活事例."}, {"role": "assistant", "content": "彝族;尔普;礼物交换;族群互动"}], ensure_ascii=False),
+                    "unsloth/Qwen3-8B-unsloth-bnb-4bit",
+                    True,
+                ]
+            ],
+            inputs=[ct_conversation, ct_model_name, ct_add_generation_prompt],
+            outputs=[ct_output],
+            fn=run_chat_template,
+        )
+
+    return locals()
+
+
+if __name__ == "__main__":
+    pass

tabs/shell_tab.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import gradio as gr
+
+from toolbox.os.command import Command
+
+
+def shell(cmd: str):
+    return Command.popen(cmd)
+
+
+def get_shell_tab():
+    with gr.TabItem("shell"):
+        shell_text = gr.Textbox(label="cmd")
+        shell_button = gr.Button("run")
+        shell_output = gr.Textbox(label="output", max_lines=100)
+
+        shell_button.click(
+            shell,
+            inputs=[shell_text, ],
+            outputs=[shell_output],
+        )
+
+        gr.Examples(
+            examples=[
+                [
+                    "echo \"CPU使用率: $(grep 'cpu ' /proc/stat | awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage \"%\"}')\""
+                ], [
+                    "echo \"内存使用: $(free -m | awk '/Mem:/ {printf \"%.1f%%\", $3/$2*100}')\""
+                ], [
+                    "echo \"内存总量: $(grep MemTotal /proc/meminfo | awk '{print $2/1024 \" MB\"}')\""
+                ], [
+                    "echo \"可用内存: $(grep MemAvailable /proc/meminfo | awk '{print $2/1024 \" MB\"}')\""
+                ], [
+                    "grep 'less' logs/info.log | tail -n 15"
+                ], [
+                    "ffmpeg -i /home/user/app/data/video/download/video.mp4 -vn -acodec libmp3lame -q:a 2 /home/user/app/data/video/download/audio.mp3"
+                ]
+            ],
+            inputs=[shell_text],
+            outputs=[shell_output],
+        )
+
+    return locals()
+
+
+if __name__ == "__main__":
+    pass