Percy3822 commited on
Commit
0aef10f
Β·
verified Β·
1 Parent(s): 6f02285

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +33 -59
app.py CHANGED
@@ -1,16 +1,14 @@
1
- import os, shutil, subprocess, zipfile, time
 
2
  from pathlib import Path
3
  import gradio as gr
4
 
5
- ROOT = Path(__file__).resolve().parent # /home/user/app
6
  DATA = ROOT / "dataset.jsonl"
7
  LOG = ROOT / "train.log"
8
  OUT = ROOT / "trained_model"
9
  ZIP = ROOT / "trained_model.zip"
10
- DONE = ROOT / "TRAIN_DONE" # <- completion flag
11
- ERRF = ROOT / "TRAIN_ERROR" # <- error flag
12
 
13
- # ---------- helpers ----------
14
  def ls_workspace() -> str:
15
  rows = []
16
  for p in sorted(ROOT.iterdir(), key=lambda x: (x.is_file(), x.name.lower())):
@@ -19,24 +17,6 @@ def ls_workspace() -> str:
19
  rows.append(f"{'[DIR]' if p.is_dir() else ' '}\t{size:>10}\t{p.name}")
20
  return "\n".join(rows) or "(empty)"
21
 
22
- def _reset_artifacts():
23
- for path in [OUT, ZIP, DONE, ERRF, LOG]:
24
- if path.is_dir():
25
- shutil.rmtree(path, ignore_errors=True)
26
- else:
27
- path.unlink(missing_ok=True)
28
-
29
- def _zip_if_ready() -> bool:
30
- """Zip OUT β†’ ZIP once DONE exists."""
31
- if DONE.exists() and OUT.exists():
32
- if ZIP.exists():
33
- ZIP.unlink()
34
- with zipfile.ZipFile(ZIP, "w", compression=zipfile.ZIP_DEFLATED) as z:
35
- for p in OUT.rglob("*"):
36
- z.write(p, arcname=p.relative_to(OUT))
37
- return ZIP.exists()
38
-
39
- # ---------- train tab callbacks ----------
40
  def upload_dataset(file):
41
  if not file:
42
  return "❌ No file selected.", ls_workspace()
@@ -44,44 +24,41 @@ def upload_dataset(file):
44
  return f"βœ… Uploaded β†’ {DATA.name}", ls_workspace()
45
 
46
  def start_training():
47
- if not DATA.exists():
48
- return "❌ Upload a JSONL first.", ls_workspace()
49
-
50
- _reset_artifacts()
51
- LOG.write_text("πŸ”₯ Training started in background…\n", encoding="utf-8")
52
 
53
  cmd = [
54
- "python", "train.py",
55
  "--dataset", str(DATA),
56
  "--output", str(OUT),
 
57
  "--model_name", "Salesforce/codegen-350M-multi",
58
  "--epochs", "1",
59
  "--batch_size", "2",
60
  "--block_size", "256",
61
  "--learning_rate", "5e-5",
62
- "--subset", "0",
63
  ]
 
64
  with open(LOG, "a", encoding="utf-8") as lf:
65
- subprocess.Popen(cmd, stdout=lf, stderr=subprocess.STDOUT)
66
 
67
- return "πŸš€ Training launched. Use Refresh buttons.", ls_workspace()
 
 
 
 
 
 
68
 
69
  def read_logs():
70
  return LOG.read_text(encoding="utf-8")[-20000:] if LOG.exists() else "⏳ Waiting…"
71
 
72
- def refresh_status_and_download():
73
- if ERRF.exists():
74
- status = f"❌ Error:\n{ERRF.read_text(encoding='utf-8')[-1200:]}"
75
- elif DONE.exists():
76
- status = "βœ… Training complete."
77
- else:
78
- status = "⏳ Training…"
79
-
80
- _zip_if_ready()
81
- files = [str(ZIP)] if ZIP.exists() else []
82
- return status, gr.Files.update(value=files, visible=bool(files)), ls_workspace()
83
 
84
- # ---------- test tab ----------
85
  def list_models():
86
  out = []
87
  for p in ROOT.iterdir():
@@ -96,17 +73,16 @@ def list_models():
96
  def import_zip(zfile):
97
  if not zfile:
98
  return "❌ No zip selected.", list_models()
99
- dest = ROOT / f"imported_{int(time.time())}"
 
100
  dest.mkdir(parents=True, exist_ok=True)
101
  with zipfile.ZipFile(zfile.name, "r") as z:
102
  z.extractall(dest)
103
  return f"βœ… Imported to {dest.name}", list_models()
104
 
105
  def generate(model_path, prompt):
106
- if not model_path:
107
- return "❌ Select a model."
108
- if not prompt or not prompt.strip():
109
- return "❌ Enter a prompt."
110
  try:
111
  from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
112
  tok = AutoTokenizer.from_pretrained(model_path, use_fast=True)
@@ -123,9 +99,8 @@ def generate(model_path, prompt):
123
  except Exception as e:
124
  return f"❌ Error: {e}"
125
 
126
- # ---------- UI ----------
127
  with gr.Blocks(title="Python AI β€” Train & Test") as app:
128
- gr.Markdown("## 🧠 Python AI β€” Train & Test\nBackground training with DONE flag β†’ reliable zip.\n")
129
 
130
  with gr.Tab("Train"):
131
  with gr.Row():
@@ -133,21 +108,20 @@ with gr.Blocks(title="Python AI β€” Train & Test") as app:
133
  ws = gr.Textbox(label="Workspace", lines=16, value=ls_workspace())
134
  up_status = gr.Textbox(label="Upload Status", interactive=False)
135
  start = gr.Button("πŸš€ Start Training", variant="primary")
136
- logs = gr.Textbox(label="πŸ“œ Logs (click Refresh)", lines=18)
137
- refresh_logs_btn = gr.Button("Refresh Logs")
138
  status = gr.Textbox(label="Status", interactive=False)
139
- downloads = gr.Files(label="πŸ“¦ Downloads (zips)", value=[], interactive=False)
140
- refresh_dl_btn = gr.Button("Refresh Status & Download")
141
 
142
  ds.change(upload_dataset, inputs=ds, outputs=[up_status, ws])
143
- start.click(start_training, outputs=[status, ws])
144
- refresh_logs_btn.click(read_logs, outputs=logs)
145
- refresh_dl_btn.click(refresh_status_and_download, outputs=[status, downloads, ws])
146
 
147
  with gr.Tab("Test"):
 
148
  refresh_btn = gr.Button("↻ Refresh Model List")
149
  model_list = gr.Dropdown(choices=list_models(), label="Available AIs", interactive=True)
150
- zip_in = gr.File(label="Or upload model .zip", file_types=[".zip"])
151
  import_status = gr.Textbox(label="Import Status", interactive=False)
152
  prompt = gr.Textbox(label="Prompt", lines=8, placeholder="### Instruction:\nPython: write a function ...\n### Response:\n")
153
  go = gr.Button("Generate")
 
1
+ # app.py
2
+ import os, shutil, subprocess, zipfile
3
  from pathlib import Path
4
  import gradio as gr
5
 
6
+ ROOT = Path(_file_).resolve().parent # /home/user/app
7
  DATA = ROOT / "dataset.jsonl"
8
  LOG = ROOT / "train.log"
9
  OUT = ROOT / "trained_model"
10
  ZIP = ROOT / "trained_model.zip"
 
 
11
 
 
12
  def ls_workspace() -> str:
13
  rows = []
14
  for p in sorted(ROOT.iterdir(), key=lambda x: (x.is_file(), x.name.lower())):
 
17
  rows.append(f"{'[DIR]' if p.is_dir() else ' '}\t{size:>10}\t{p.name}")
18
  return "\n".join(rows) or "(empty)"
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  def upload_dataset(file):
21
  if not file:
22
  return "❌ No file selected.", ls_workspace()
 
24
  return f"βœ… Uploaded β†’ {DATA.name}", ls_workspace()
25
 
26
  def start_training():
27
+ # clean previous artifacts
28
+ if OUT.exists(): shutil.rmtree(OUT, ignore_errors=True)
29
+ if ZIP.exists(): ZIP.unlink(missing_ok=True)
30
+ LOG.write_text("πŸ”₯ Training started…\n", encoding="utf-8")
 
31
 
32
  cmd = [
33
+ "python", str(ROOT / "train.py"),
34
  "--dataset", str(DATA),
35
  "--output", str(OUT),
36
+ "--zip_path", str(ZIP),
37
  "--model_name", "Salesforce/codegen-350M-multi",
38
  "--epochs", "1",
39
  "--batch_size", "2",
40
  "--block_size", "256",
41
  "--learning_rate", "5e-5",
 
42
  ]
43
+ # run training (blocking) and capture logs
44
  with open(LOG, "a", encoding="utf-8") as lf:
45
+ code = subprocess.Popen(cmd, stdout=lf, stderr=subprocess.STDOUT).wait()
46
 
47
+ # after process exits, show result
48
+ if code == 0 and ZIP.exists():
49
+ info = f"βœ… Training complete. Saved: {OUT.name} | Zip: {ZIP.name}"
50
+ return info, gr.File.update(value=str(ZIP), visible=True), ls_workspace(), read_logs()
51
+ else:
52
+ info = f"❌ Training failed (exit {code}). See logs."
53
+ return info, gr.File.update(visible=False), ls_workspace(), read_logs()
54
 
55
  def read_logs():
56
  return LOG.read_text(encoding="utf-8")[-20000:] if LOG.exists() else "⏳ Waiting…"
57
 
58
+ def refresh_download():
59
+ return gr.File.update(value=str(ZIP), visible=ZIP.exists()), ls_workspace()
 
 
 
 
 
 
 
 
 
60
 
61
+ # ---------------- Test tab ----------------
62
  def list_models():
63
  out = []
64
  for p in ROOT.iterdir():
 
73
  def import_zip(zfile):
74
  if not zfile:
75
  return "❌ No zip selected.", list_models()
76
+ dest = ROOT / f"imported_model"
77
+ if dest.exists(): shutil.rmtree(dest, ignore_errors=True)
78
  dest.mkdir(parents=True, exist_ok=True)
79
  with zipfile.ZipFile(zfile.name, "r") as z:
80
  z.extractall(dest)
81
  return f"βœ… Imported to {dest.name}", list_models()
82
 
83
  def generate(model_path, prompt):
84
+ if not model_path: return "❌ Select a model."
85
+ if not prompt or not prompt.strip(): return "❌ Enter a prompt."
 
 
86
  try:
87
  from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
88
  tok = AutoTokenizer.from_pretrained(model_path, use_fast=True)
 
99
  except Exception as e:
100
  return f"❌ Error: {e}"
101
 
 
102
  with gr.Blocks(title="Python AI β€” Train & Test") as app:
103
+ gr.Markdown("## 🧠 Python AI β€” Train & Test (simple + reliable)\nTrainer zips the model itself. UI just shows the zip.\n")
104
 
105
  with gr.Tab("Train"):
106
  with gr.Row():
 
108
  ws = gr.Textbox(label="Workspace", lines=16, value=ls_workspace())
109
  up_status = gr.Textbox(label="Upload Status", interactive=False)
110
  start = gr.Button("πŸš€ Start Training", variant="primary")
111
+ logs = gr.Textbox(label="πŸ“œ Training Logs", lines=18, value=read_logs())
 
112
  status = gr.Textbox(label="Status", interactive=False)
113
+ download_file = gr.File(label="πŸ“¦ trained_model.zip", visible=ZIP.exists())
114
+ refresh_dl_btn = gr.Button("Refresh Download")
115
 
116
  ds.change(upload_dataset, inputs=ds, outputs=[up_status, ws])
117
+ start.click(start_training, outputs=[status, download_file, ws, logs])
118
+ refresh_dl_btn.click(refresh_download, outputs=[download_file, ws])
 
119
 
120
  with gr.Tab("Test"):
121
+ gr.Markdown("### Choose a model folder or upload a .zip, then prompt it")
122
  refresh_btn = gr.Button("↻ Refresh Model List")
123
  model_list = gr.Dropdown(choices=list_models(), label="Available AIs", interactive=True)
124
+ zip_in = gr.File(label="Or upload a model .zip", file_types=[".zip"])
125
  import_status = gr.Textbox(label="Import Status", interactive=False)
126
  prompt = gr.Textbox(label="Prompt", lines=8, placeholder="### Instruction:\nPython: write a function ...\n### Response:\n")
127
  go = gr.Button("Generate")