Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,16 +1,14 @@
|
|
| 1 |
-
|
|
|
|
| 2 |
from pathlib import Path
|
| 3 |
import gradio as gr
|
| 4 |
|
| 5 |
-
ROOT = Path(
|
| 6 |
DATA = ROOT / "dataset.jsonl"
|
| 7 |
LOG = ROOT / "train.log"
|
| 8 |
OUT = ROOT / "trained_model"
|
| 9 |
ZIP = ROOT / "trained_model.zip"
|
| 10 |
-
DONE = ROOT / "TRAIN_DONE" # <- completion flag
|
| 11 |
-
ERRF = ROOT / "TRAIN_ERROR" # <- error flag
|
| 12 |
|
| 13 |
-
# ---------- helpers ----------
|
| 14 |
def ls_workspace() -> str:
|
| 15 |
rows = []
|
| 16 |
for p in sorted(ROOT.iterdir(), key=lambda x: (x.is_file(), x.name.lower())):
|
|
@@ -19,24 +17,6 @@ def ls_workspace() -> str:
|
|
| 19 |
rows.append(f"{'[DIR]' if p.is_dir() else ' '}\t{size:>10}\t{p.name}")
|
| 20 |
return "\n".join(rows) or "(empty)"
|
| 21 |
|
| 22 |
-
def _reset_artifacts():
|
| 23 |
-
for path in [OUT, ZIP, DONE, ERRF, LOG]:
|
| 24 |
-
if path.is_dir():
|
| 25 |
-
shutil.rmtree(path, ignore_errors=True)
|
| 26 |
-
else:
|
| 27 |
-
path.unlink(missing_ok=True)
|
| 28 |
-
|
| 29 |
-
def _zip_if_ready() -> bool:
|
| 30 |
-
"""Zip OUT β ZIP once DONE exists."""
|
| 31 |
-
if DONE.exists() and OUT.exists():
|
| 32 |
-
if ZIP.exists():
|
| 33 |
-
ZIP.unlink()
|
| 34 |
-
with zipfile.ZipFile(ZIP, "w", compression=zipfile.ZIP_DEFLATED) as z:
|
| 35 |
-
for p in OUT.rglob("*"):
|
| 36 |
-
z.write(p, arcname=p.relative_to(OUT))
|
| 37 |
-
return ZIP.exists()
|
| 38 |
-
|
| 39 |
-
# ---------- train tab callbacks ----------
|
| 40 |
def upload_dataset(file):
|
| 41 |
if not file:
|
| 42 |
return "β No file selected.", ls_workspace()
|
|
@@ -44,44 +24,41 @@ def upload_dataset(file):
|
|
| 44 |
return f"β
Uploaded β {DATA.name}", ls_workspace()
|
| 45 |
|
| 46 |
def start_training():
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
LOG.write_text("π₯ Training started in backgroundβ¦\n", encoding="utf-8")
|
| 52 |
|
| 53 |
cmd = [
|
| 54 |
-
"python", "train.py",
|
| 55 |
"--dataset", str(DATA),
|
| 56 |
"--output", str(OUT),
|
|
|
|
| 57 |
"--model_name", "Salesforce/codegen-350M-multi",
|
| 58 |
"--epochs", "1",
|
| 59 |
"--batch_size", "2",
|
| 60 |
"--block_size", "256",
|
| 61 |
"--learning_rate", "5e-5",
|
| 62 |
-
"--subset", "0",
|
| 63 |
]
|
|
|
|
| 64 |
with open(LOG, "a", encoding="utf-8") as lf:
|
| 65 |
-
subprocess.Popen(cmd, stdout=lf, stderr=subprocess.STDOUT)
|
| 66 |
|
| 67 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
|
| 69 |
def read_logs():
|
| 70 |
return LOG.read_text(encoding="utf-8")[-20000:] if LOG.exists() else "β³ Waitingβ¦"
|
| 71 |
|
| 72 |
-
def
|
| 73 |
-
|
| 74 |
-
status = f"β Error:\n{ERRF.read_text(encoding='utf-8')[-1200:]}"
|
| 75 |
-
elif DONE.exists():
|
| 76 |
-
status = "β
Training complete."
|
| 77 |
-
else:
|
| 78 |
-
status = "β³ Trainingβ¦"
|
| 79 |
-
|
| 80 |
-
_zip_if_ready()
|
| 81 |
-
files = [str(ZIP)] if ZIP.exists() else []
|
| 82 |
-
return status, gr.Files.update(value=files, visible=bool(files)), ls_workspace()
|
| 83 |
|
| 84 |
-
#
|
| 85 |
def list_models():
|
| 86 |
out = []
|
| 87 |
for p in ROOT.iterdir():
|
|
@@ -96,17 +73,16 @@ def list_models():
|
|
| 96 |
def import_zip(zfile):
|
| 97 |
if not zfile:
|
| 98 |
return "β No zip selected.", list_models()
|
| 99 |
-
dest = ROOT / f"
|
|
|
|
| 100 |
dest.mkdir(parents=True, exist_ok=True)
|
| 101 |
with zipfile.ZipFile(zfile.name, "r") as z:
|
| 102 |
z.extractall(dest)
|
| 103 |
return f"β
Imported to {dest.name}", list_models()
|
| 104 |
|
| 105 |
def generate(model_path, prompt):
|
| 106 |
-
if not model_path:
|
| 107 |
-
|
| 108 |
-
if not prompt or not prompt.strip():
|
| 109 |
-
return "β Enter a prompt."
|
| 110 |
try:
|
| 111 |
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
|
| 112 |
tok = AutoTokenizer.from_pretrained(model_path, use_fast=True)
|
|
@@ -123,9 +99,8 @@ def generate(model_path, prompt):
|
|
| 123 |
except Exception as e:
|
| 124 |
return f"β Error: {e}"
|
| 125 |
|
| 126 |
-
# ---------- UI ----------
|
| 127 |
with gr.Blocks(title="Python AI β Train & Test") as app:
|
| 128 |
-
gr.Markdown("## π§ Python AI β Train & Test\
|
| 129 |
|
| 130 |
with gr.Tab("Train"):
|
| 131 |
with gr.Row():
|
|
@@ -133,21 +108,20 @@ with gr.Blocks(title="Python AI β Train & Test") as app:
|
|
| 133 |
ws = gr.Textbox(label="Workspace", lines=16, value=ls_workspace())
|
| 134 |
up_status = gr.Textbox(label="Upload Status", interactive=False)
|
| 135 |
start = gr.Button("π Start Training", variant="primary")
|
| 136 |
-
logs = gr.Textbox(label="π Logs
|
| 137 |
-
refresh_logs_btn = gr.Button("Refresh Logs")
|
| 138 |
status = gr.Textbox(label="Status", interactive=False)
|
| 139 |
-
|
| 140 |
-
refresh_dl_btn = gr.Button("Refresh
|
| 141 |
|
| 142 |
ds.change(upload_dataset, inputs=ds, outputs=[up_status, ws])
|
| 143 |
-
start.click(start_training, outputs=[status, ws])
|
| 144 |
-
|
| 145 |
-
refresh_dl_btn.click(refresh_status_and_download, outputs=[status, downloads, ws])
|
| 146 |
|
| 147 |
with gr.Tab("Test"):
|
|
|
|
| 148 |
refresh_btn = gr.Button("β» Refresh Model List")
|
| 149 |
model_list = gr.Dropdown(choices=list_models(), label="Available AIs", interactive=True)
|
| 150 |
-
zip_in = gr.File(label="Or upload model .zip", file_types=[".zip"])
|
| 151 |
import_status = gr.Textbox(label="Import Status", interactive=False)
|
| 152 |
prompt = gr.Textbox(label="Prompt", lines=8, placeholder="### Instruction:\nPython: write a function ...\n### Response:\n")
|
| 153 |
go = gr.Button("Generate")
|
|
|
|
| 1 |
+
# app.py
|
| 2 |
+
import os, shutil, subprocess, zipfile
|
| 3 |
from pathlib import Path
|
| 4 |
import gradio as gr
|
| 5 |
|
| 6 |
+
ROOT = Path(_file_).resolve().parent # /home/user/app
|
| 7 |
DATA = ROOT / "dataset.jsonl"
|
| 8 |
LOG = ROOT / "train.log"
|
| 9 |
OUT = ROOT / "trained_model"
|
| 10 |
ZIP = ROOT / "trained_model.zip"
|
|
|
|
|
|
|
| 11 |
|
|
|
|
| 12 |
def ls_workspace() -> str:
|
| 13 |
rows = []
|
| 14 |
for p in sorted(ROOT.iterdir(), key=lambda x: (x.is_file(), x.name.lower())):
|
|
|
|
| 17 |
rows.append(f"{'[DIR]' if p.is_dir() else ' '}\t{size:>10}\t{p.name}")
|
| 18 |
return "\n".join(rows) or "(empty)"
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
def upload_dataset(file):
|
| 21 |
if not file:
|
| 22 |
return "β No file selected.", ls_workspace()
|
|
|
|
| 24 |
return f"β
Uploaded β {DATA.name}", ls_workspace()
|
| 25 |
|
| 26 |
def start_training():
|
| 27 |
+
# clean previous artifacts
|
| 28 |
+
if OUT.exists(): shutil.rmtree(OUT, ignore_errors=True)
|
| 29 |
+
if ZIP.exists(): ZIP.unlink(missing_ok=True)
|
| 30 |
+
LOG.write_text("π₯ Training startedβ¦\n", encoding="utf-8")
|
|
|
|
| 31 |
|
| 32 |
cmd = [
|
| 33 |
+
"python", str(ROOT / "train.py"),
|
| 34 |
"--dataset", str(DATA),
|
| 35 |
"--output", str(OUT),
|
| 36 |
+
"--zip_path", str(ZIP),
|
| 37 |
"--model_name", "Salesforce/codegen-350M-multi",
|
| 38 |
"--epochs", "1",
|
| 39 |
"--batch_size", "2",
|
| 40 |
"--block_size", "256",
|
| 41 |
"--learning_rate", "5e-5",
|
|
|
|
| 42 |
]
|
| 43 |
+
# run training (blocking) and capture logs
|
| 44 |
with open(LOG, "a", encoding="utf-8") as lf:
|
| 45 |
+
code = subprocess.Popen(cmd, stdout=lf, stderr=subprocess.STDOUT).wait()
|
| 46 |
|
| 47 |
+
# after process exits, show result
|
| 48 |
+
if code == 0 and ZIP.exists():
|
| 49 |
+
info = f"β
Training complete. Saved: {OUT.name} | Zip: {ZIP.name}"
|
| 50 |
+
return info, gr.File.update(value=str(ZIP), visible=True), ls_workspace(), read_logs()
|
| 51 |
+
else:
|
| 52 |
+
info = f"β Training failed (exit {code}). See logs."
|
| 53 |
+
return info, gr.File.update(visible=False), ls_workspace(), read_logs()
|
| 54 |
|
| 55 |
def read_logs():
|
| 56 |
return LOG.read_text(encoding="utf-8")[-20000:] if LOG.exists() else "β³ Waitingβ¦"
|
| 57 |
|
| 58 |
+
def refresh_download():
|
| 59 |
+
return gr.File.update(value=str(ZIP), visible=ZIP.exists()), ls_workspace()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
|
| 61 |
+
# ---------------- Test tab ----------------
|
| 62 |
def list_models():
|
| 63 |
out = []
|
| 64 |
for p in ROOT.iterdir():
|
|
|
|
| 73 |
def import_zip(zfile):
|
| 74 |
if not zfile:
|
| 75 |
return "β No zip selected.", list_models()
|
| 76 |
+
dest = ROOT / f"imported_model"
|
| 77 |
+
if dest.exists(): shutil.rmtree(dest, ignore_errors=True)
|
| 78 |
dest.mkdir(parents=True, exist_ok=True)
|
| 79 |
with zipfile.ZipFile(zfile.name, "r") as z:
|
| 80 |
z.extractall(dest)
|
| 81 |
return f"β
Imported to {dest.name}", list_models()
|
| 82 |
|
| 83 |
def generate(model_path, prompt):
|
| 84 |
+
if not model_path: return "β Select a model."
|
| 85 |
+
if not prompt or not prompt.strip(): return "β Enter a prompt."
|
|
|
|
|
|
|
| 86 |
try:
|
| 87 |
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
|
| 88 |
tok = AutoTokenizer.from_pretrained(model_path, use_fast=True)
|
|
|
|
| 99 |
except Exception as e:
|
| 100 |
return f"β Error: {e}"
|
| 101 |
|
|
|
|
| 102 |
with gr.Blocks(title="Python AI β Train & Test") as app:
|
| 103 |
+
gr.Markdown("## π§ Python AI β Train & Test (simple + reliable)\nTrainer zips the model itself. UI just shows the zip.\n")
|
| 104 |
|
| 105 |
with gr.Tab("Train"):
|
| 106 |
with gr.Row():
|
|
|
|
| 108 |
ws = gr.Textbox(label="Workspace", lines=16, value=ls_workspace())
|
| 109 |
up_status = gr.Textbox(label="Upload Status", interactive=False)
|
| 110 |
start = gr.Button("π Start Training", variant="primary")
|
| 111 |
+
logs = gr.Textbox(label="π Training Logs", lines=18, value=read_logs())
|
|
|
|
| 112 |
status = gr.Textbox(label="Status", interactive=False)
|
| 113 |
+
download_file = gr.File(label="π¦ trained_model.zip", visible=ZIP.exists())
|
| 114 |
+
refresh_dl_btn = gr.Button("Refresh Download")
|
| 115 |
|
| 116 |
ds.change(upload_dataset, inputs=ds, outputs=[up_status, ws])
|
| 117 |
+
start.click(start_training, outputs=[status, download_file, ws, logs])
|
| 118 |
+
refresh_dl_btn.click(refresh_download, outputs=[download_file, ws])
|
|
|
|
| 119 |
|
| 120 |
with gr.Tab("Test"):
|
| 121 |
+
gr.Markdown("### Choose a model folder or upload a .zip, then prompt it")
|
| 122 |
refresh_btn = gr.Button("β» Refresh Model List")
|
| 123 |
model_list = gr.Dropdown(choices=list_models(), label="Available AIs", interactive=True)
|
| 124 |
+
zip_in = gr.File(label="Or upload a model .zip", file_types=[".zip"])
|
| 125 |
import_status = gr.Textbox(label="Import Status", interactive=False)
|
| 126 |
prompt = gr.Textbox(label="Prompt", lines=8, placeholder="### Instruction:\nPython: write a function ...\n### Response:\n")
|
| 127 |
go = gr.Button("Generate")
|