Dishaaa25 commited on
Commit
f77210b
·
1 Parent(s): 1bebb32

Revert "Add model playground demo UI"

Browse files

This reverts commit a000f2ac127075ec4693648b000a950d19c03e74.

Files changed (4) hide show
  1. .DS_Store +0 -0
  2. env/app.py +64 -309
  3. server/app.py +0 -2
  4. server/runtime.py +23 -179
.DS_Store DELETED
Binary file (6.15 kB)
 
env/app.py CHANGED
@@ -1,6 +1,5 @@
1
  from __future__ import annotations
2
 
3
- from html import escape
4
  from uuid import uuid4
5
 
6
  import gradio as gr
@@ -12,15 +11,6 @@ from server.runtime import SpaceTrainingManager
12
  TRAINING_MANAGER = SpaceTrainingManager()
13
  SESSIONS: dict[str, AdaptEnvironment] = {}
14
 
15
- PLAYGROUND_DEFAULT_PROBLEM = (
16
- "Given an array of integers, return the length of the longest contiguous subarray "
17
- "whose sum is divisible by k."
18
- )
19
- PLAYGROUND_DEFAULT_INPUT = (
20
- "The first line contains two integers n and k. The second line contains n space-separated integers."
21
- )
22
- PLAYGROUND_DEFAULT_CONSTRAINTS = "1 <= n <= 2 * 10^5, 1 <= k <= 10^9, array values fit in 32-bit signed integers."
23
-
24
 
25
  def _get_env(session_id: str | None) -> AdaptEnvironment:
26
  if not session_id or session_id not in SESSIONS:
@@ -53,8 +43,6 @@ def sample_problem(problem_id: str, difficulty: str) -> tuple[str, str, str, str
53
  "",
54
  payload,
55
  )
56
-
57
-
58
  def evaluate_submission(session_id: str, code: str) -> tuple[str, str, str, dict]:
59
  env = _get_env(session_id)
60
  observation = env.step(AdaptAction(session_id=env.session_id, code=code))
@@ -107,314 +95,81 @@ def model_attempt(session_id: str) -> tuple[str, str, str, dict]:
107
  return evaluate_submission(session_id, generation["code"])
108
 
109
 
110
- def _format_accuracy(value: float | None) -> str:
111
- if value is None:
112
- return "Unavailable"
113
- return f"{value * 100:.1f}%"
114
-
115
-
116
- def _format_progress(completed_steps: int, total_steps: int, progress_ratio: float) -> str:
117
- if total_steps > 0:
118
- return f"{completed_steps}/{total_steps} steps ({progress_ratio * 100:.1f}%)"
119
- return "No active training run"
120
-
121
-
122
- def _render_metrics_card(metrics: dict) -> str:
123
- overall_accuracy = metrics.get("overall_accuracy")
124
- baseline_accuracy = metrics.get("baseline_accuracy")
125
- live_pass_rate = metrics.get("live_pass_rate")
126
- metric_source = metrics.get("metric_source", "unavailable")
127
- training_status = escape(str(metrics.get("training_status", "unknown")).replace("_", " ").title())
128
- phase = escape(str(metrics.get("phase", "idle")).replace("_", " ").title())
129
- progress = escape(
130
- _format_progress(
131
- int(metrics.get("completed_steps", 0) or 0),
132
- int(metrics.get("total_steps", 0) or 0),
133
- float(metrics.get("progress_ratio", 0.0) or 0.0),
134
- )
135
- )
136
- source_label = {
137
- "trained_eval": "Final evaluation metric",
138
- "rolling_pass_rate": "Live rolling pass rate",
139
- "unavailable": "Metric unavailable",
140
- }.get(metric_source, "Metric unavailable")
141
- source_label = escape(source_label)
142
- live_pass_rate_text = escape(_format_accuracy(live_pass_rate))
143
- overall_accuracy_text = escape(_format_accuracy(overall_accuracy))
144
- baseline_accuracy_text = escape(_format_accuracy(baseline_accuracy))
145
-
146
- return f"""
147
- <div class="panel metrics-card">
148
- <div class="metrics-header">
149
- <div>
150
- <div class="metrics-eyebrow">Live Demo Dashboard</div>
151
- <h3>Model Accuracy Tracker</h3>
152
- </div>
153
- <div class="metrics-chip">{source_label}</div>
154
- </div>
155
- <div class="metrics-grid">
156
- <div class="metric-tile">
157
- <span>Overall Accuracy</span>
158
- <strong>{overall_accuracy_text}</strong>
159
- </div>
160
- <div class="metric-tile">
161
- <span>Base Accuracy</span>
162
- <strong>{baseline_accuracy_text}</strong>
163
- </div>
164
- <div class="metric-tile">
165
- <span>Live Pass Rate</span>
166
- <strong>{live_pass_rate_text}</strong>
167
- </div>
168
- <div class="metric-tile">
169
- <span>Training Phase</span>
170
- <strong>{phase}</strong>
171
- </div>
172
- </div>
173
- <div class="metrics-footer">
174
- <span>Status: <strong>{training_status}</strong></span>
175
- <span>Progress: <strong>{progress}</strong></span>
176
- </div>
177
- </div>
178
- """
179
-
180
-
181
- def playground_metrics_view() -> str:
182
- payload = TRAINING_MANAGER.status_payload()
183
- metrics = payload.get("demo_metrics", {})
184
- return _render_metrics_card(metrics if isinstance(metrics, dict) else {})
185
-
186
-
187
- def _model_badge(result: dict | None, label: str, error: str | None = None) -> str:
188
- if error:
189
- return f"### {label}\nStatus: {error}"
190
- if not result:
191
- return f"### {label}\nStatus: No generation yet."
192
- model = result.get("model", {}) if isinstance(result.get("model"), dict) else {}
193
- effective_model = str(result.get("effective_model", model.get("active_model_kind", "unavailable"))).replace("_", " ")
194
- source = model.get("source_repo_id") or model.get("base_model_name") or model.get("local_path") or "unknown source"
195
- details = [f"Status: {effective_model.title()}"]
196
- if model.get("fallback_reason"):
197
- details.append("Fallback: trained model unavailable, using base model")
198
- details.append(f"Source: {source}")
199
- if model.get("revision"):
200
- details.append(f"Revision: {model['revision']}")
201
- return f"### {label}\n" + "\n".join(details)
202
-
203
-
204
- def compare_models(problem: str, input_format: str, constraints: str) -> tuple[str, str, str, str]:
205
- if not problem.strip():
206
- message = "Please enter a problem statement before generating solutions."
207
- return "", "", _model_badge(None, "Base Model", message), _model_badge(None, "ADAPT-Trained Model", message)
208
- if not input_format.strip():
209
- message = "Please provide the input format so the model prompt is well-formed."
210
- return "", "", _model_badge(None, "Base Model", message), _model_badge(None, "ADAPT-Trained Model", message)
211
- if not constraints.strip():
212
- message = "Please provide constraints so the model can target the expected solution shape."
213
- return "", "", _model_badge(None, "Base Model", message), _model_badge(None, "ADAPT-Trained Model", message)
214
-
215
- base_result: dict | None = None
216
- trained_result: dict | None = None
217
- base_error: str | None = None
218
- trained_error: str | None = None
219
-
220
- try:
221
- base_result = TRAINING_MANAGER.generate_code(
222
- problem=problem,
223
- input_format=input_format,
224
- constraints=constraints,
225
- target_model="base",
226
- )
227
- except Exception as exc:
228
- base_error = str(exc)
229
-
230
- try:
231
- trained_result = TRAINING_MANAGER.generate_code(
232
- problem=problem,
233
- input_format=input_format,
234
- constraints=constraints,
235
- target_model="current",
236
- )
237
- except Exception as exc:
238
- trained_error = str(exc)
239
-
240
- return (
241
- base_result.get("code", "") if base_result else "",
242
- trained_result.get("code", "") if trained_result else "",
243
- _model_badge(base_result, "Base Model", base_error),
244
- _model_badge(trained_result, "ADAPT-Trained Model", trained_error),
245
- )
246
-
247
-
248
  with gr.Blocks(
249
  title="ADAPT DSA Tutor Demo",
250
  css="""
251
  .panel {border: 1px solid #d7d3c9; border-radius: 18px; background: #fffaf2;}
252
  .hero {background: linear-gradient(135deg, #f7eedb, #f3f8ef); border-radius: 22px; padding: 18px;}
253
- .metrics-card {padding: 18px;}
254
- .metrics-header {display: flex; justify-content: space-between; align-items: flex-start; gap: 12px; margin-bottom: 14px;}
255
- .metrics-header h3 {margin: 4px 0 0; font-size: 1.2rem;}
256
- .metrics-eyebrow {font-size: 0.8rem; letter-spacing: 0.08em; text-transform: uppercase; color: #816b45;}
257
- .metrics-chip {background: #efe3c6; color: #6f5529; border-radius: 999px; padding: 6px 10px; font-size: 0.85rem;}
258
- .metrics-grid {display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 12px; margin-bottom: 14px;}
259
- .metric-tile {background: #fff; border: 1px solid #eadfc9; border-radius: 14px; padding: 12px;}
260
- .metric-tile span {display: block; font-size: 0.82rem; color: #7a6541; margin-bottom: 6px;}
261
- .metric-tile strong {font-size: 1.1rem; color: #2f2412;}
262
- .metrics-footer {display: flex; justify-content: space-between; gap: 12px; flex-wrap: wrap; color: #5c4c30;}
263
- @media (max-width: 900px) {
264
- .metrics-grid {grid-template-columns: repeat(2, minmax(0, 1fr));}
265
- }
266
- @media (max-width: 640px) {
267
- .metrics-grid {grid-template-columns: minmax(0, 1fr);}
268
- }
269
  """,
270
  ) as demo:
271
- with gr.Tab("Verifier Sandbox"):
272
- session_id = gr.Textbox(label="Session ID", interactive=False)
273
- state_payload = gr.JSON(label="Observation Payload")
274
-
275
- gr.Markdown(
276
- """
277
- # ADAPT DSA Tutor
278
- Sample a problem, inspect the verifier feedback, and compare your repair attempt with the currently loaded model path.
279
- """,
280
- elem_classes=["hero"],
281
- )
282
-
283
- with gr.Row():
284
- problem_id = gr.Dropdown(
285
- choices=[
286
- "",
287
- "sum_even_numbers",
288
- "range_span",
289
- "count_vowels",
290
- "max_consecutive_ones",
291
- "fizzbuzz_variant",
292
- "running_total",
293
- "count_local_peaks",
294
- "longest_non_decreasing_run",
295
- "two_sum_count",
296
- "max_subarray_sum",
297
- "group_anagrams_count",
298
- "balanced_brackets",
299
- "matrix_diagonal_sum",
300
- "smallest_most_frequent",
301
- "reverse_words",
302
- "longest_common_subsequence",
303
- "word_ladder_steps",
304
- "merge_intervals",
305
- "min_coins",
306
- "rotate_matrix_90",
307
- ],
308
- value="",
309
- label="Problem Family",
310
- info="Leave blank to sample automatically.",
311
- )
312
- difficulty = gr.Radio(choices=["easy", "medium", "hard"], value="easy", label="Difficulty")
313
- sample_btn = gr.Button("Sample Problem", variant="primary")
314
-
315
- problem_view = gr.Markdown(elem_classes=["panel"])
316
- with gr.Row():
317
- code = gr.Textbox(
318
- label="Python Submission",
319
- lines=18,
320
- max_lines=24,
321
- placeholder="Write code that reads stdin and prints stdout.",
322
- )
323
- with gr.Column():
324
- feedback = gr.Textbox(label="Verifier Feedback", lines=14)
325
- status = gr.Textbox(label="Scorecard", lines=4)
326
- with gr.Row():
327
- verify_btn = gr.Button("Verify Submission", variant="primary")
328
- model_btn = gr.Button("Run Current Model", variant="secondary")
329
-
330
- sample_btn.click(
331
- fn=sample_problem,
332
- inputs=[problem_id, difficulty],
333
- outputs=[session_id, problem_view, feedback, code, state_payload],
334
- )
335
- verify_btn.click(
336
- fn=evaluate_submission,
337
- inputs=[session_id, code],
338
- outputs=[feedback, status, code, state_payload],
339
- )
340
- model_btn.click(
341
- fn=model_attempt,
342
- inputs=[session_id],
343
- outputs=[feedback, status, code, state_payload],
344
- )
345
-
346
- with gr.Tab("Model Playground"):
347
- gr.Markdown(
348
- """
349
- # Model Playground
350
- Paste a problem and compare how the base model stacks up against the current ADAPT-powered solver while training metrics refresh live below.
351
- """,
352
- elem_classes=["hero"],
353
- )
354
- playground_timer = gr.Timer(value=5)
355
-
356
- problem_text = gr.Textbox(
357
- label="Problem",
358
- lines=8,
359
- value=PLAYGROUND_DEFAULT_PROBLEM,
360
- placeholder="Paste the full problem statement here.",
361
- )
362
- with gr.Row():
363
- input_format_text = gr.Textbox(
364
- label="Input Format",
365
- lines=5,
366
- value=PLAYGROUND_DEFAULT_INPUT,
367
- placeholder="Describe how stdin is formatted.",
368
- )
369
- constraints_text = gr.Textbox(
370
- label="Constraints",
371
- lines=5,
372
- value=PLAYGROUND_DEFAULT_CONSTRAINTS,
373
- placeholder="Add the important bounds and edge constraints.",
374
- )
375
- with gr.Row():
376
- generate_btn = gr.Button("Generate Solutions", variant="primary")
377
- clear_btn = gr.Button("Clear", variant="secondary")
378
-
379
- with gr.Row():
380
- with gr.Column():
381
- base_code = gr.Code(label="Base Model Output", language="python", lines=20)
382
- base_status = gr.Markdown(_model_badge(None, "Base Model"))
383
- with gr.Column():
384
- trained_code = gr.Code(label="ADAPT-Trained Model Output", language="python", lines=20)
385
- trained_status = gr.Markdown(_model_badge(None, "ADAPT-Trained Model"))
386
-
387
- metrics_dashboard = gr.HTML(playground_metrics_view())
388
 
389
- generate_btn.click(
390
- fn=compare_models,
391
- inputs=[problem_text, input_format_text, constraints_text],
392
- outputs=[base_code, trained_code, base_status, trained_status],
393
- )
394
- clear_btn.click(
395
- fn=lambda: (
396
- "",
397
  "",
398
- "",
399
- "",
400
- "",
401
- _model_badge(None, "Base Model"),
402
- _model_badge(None, "ADAPT-Trained Model"),
403
- playground_metrics_view(),
404
- ),
405
- outputs=[
406
- problem_text,
407
- input_format_text,
408
- constraints_text,
409
- base_code,
410
- trained_code,
411
- base_status,
412
- trained_status,
413
- metrics_dashboard,
 
 
 
 
414
  ],
 
 
 
415
  )
416
- playground_timer.tick(fn=playground_metrics_view, outputs=[metrics_dashboard])
417
- demo.load(fn=playground_metrics_view, outputs=[metrics_dashboard])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
418
 
419
 
420
  if __name__ == "__main__":
 
1
  from __future__ import annotations
2
 
 
3
  from uuid import uuid4
4
 
5
  import gradio as gr
 
11
  TRAINING_MANAGER = SpaceTrainingManager()
12
  SESSIONS: dict[str, AdaptEnvironment] = {}
13
 
 
 
 
 
 
 
 
 
 
14
 
15
  def _get_env(session_id: str | None) -> AdaptEnvironment:
16
  if not session_id or session_id not in SESSIONS:
 
43
  "",
44
  payload,
45
  )
 
 
46
  def evaluate_submission(session_id: str, code: str) -> tuple[str, str, str, dict]:
47
  env = _get_env(session_id)
48
  observation = env.step(AdaptAction(session_id=env.session_id, code=code))
 
95
  return evaluate_submission(session_id, generation["code"])
96
 
97
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  with gr.Blocks(
99
  title="ADAPT DSA Tutor Demo",
100
  css="""
101
  .panel {border: 1px solid #d7d3c9; border-radius: 18px; background: #fffaf2;}
102
  .hero {background: linear-gradient(135deg, #f7eedb, #f3f8ef); border-radius: 22px; padding: 18px;}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  """,
104
  ) as demo:
105
+ session_id = gr.Textbox(label="Session ID", interactive=False)
106
+ state_payload = gr.JSON(label="Observation Payload")
107
+
108
+ gr.Markdown(
109
+ """
110
+ # ADAPT DSA Tutor
111
+ Sample a problem, inspect the verifier feedback, and compare your repair attempt with the currently loaded model path.
112
+ """,
113
+ elem_classes=["hero"],
114
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
 
116
+ with gr.Row():
117
+ problem_id = gr.Dropdown(
118
+ choices=[
 
 
 
 
 
119
  "",
120
+ "sum_even_numbers",
121
+ "range_span",
122
+ "count_vowels",
123
+ "max_consecutive_ones",
124
+ "fizzbuzz_variant",
125
+ "running_total",
126
+ "count_local_peaks",
127
+ "longest_non_decreasing_run",
128
+ "two_sum_count",
129
+ "max_subarray_sum",
130
+ "group_anagrams_count",
131
+ "balanced_brackets",
132
+ "matrix_diagonal_sum",
133
+ "smallest_most_frequent",
134
+ "reverse_words",
135
+ "longest_common_subsequence",
136
+ "word_ladder_steps",
137
+ "merge_intervals",
138
+ "min_coins",
139
+ "rotate_matrix_90",
140
  ],
141
+ value="",
142
+ label="Problem Family",
143
+ info="Leave blank to sample automatically.",
144
  )
145
+ difficulty = gr.Radio(choices=["easy", "medium", "hard"], value="easy", label="Difficulty")
146
+ sample_btn = gr.Button("Sample Problem", variant="primary")
147
+
148
+ problem_view = gr.Markdown(elem_classes=["panel"])
149
+ with gr.Row():
150
+ code = gr.Textbox(label="Python Submission", lines=18, max_lines=24, placeholder="Write code that reads stdin and prints stdout.")
151
+ with gr.Column():
152
+ feedback = gr.Textbox(label="Verifier Feedback", lines=14)
153
+ status = gr.Textbox(label="Scorecard", lines=4)
154
+ with gr.Row():
155
+ verify_btn = gr.Button("Verify Submission", variant="primary")
156
+ model_btn = gr.Button("Run Current Model", variant="secondary")
157
+
158
+ sample_btn.click(
159
+ fn=sample_problem,
160
+ inputs=[problem_id, difficulty],
161
+ outputs=[session_id, problem_view, feedback, code, state_payload],
162
+ )
163
+ verify_btn.click(
164
+ fn=evaluate_submission,
165
+ inputs=[session_id, code],
166
+ outputs=[feedback, status, code, state_payload],
167
+ )
168
+ model_btn.click(
169
+ fn=model_attempt,
170
+ inputs=[session_id],
171
+ outputs=[feedback, status, code, state_payload],
172
+ )
173
 
174
 
175
  if __name__ == "__main__":
server/app.py CHANGED
@@ -99,7 +99,6 @@ class GenerateCodeRequest(BaseModel):
99
  attempt_number: int = 1
100
  max_steps: int = 1
101
  max_new_tokens: int = 512
102
- target_model: str = "current"
103
 
104
 
105
  def _metadata() -> dict[str, Any]:
@@ -254,7 +253,6 @@ def generate_code(request: GenerateCodeRequest) -> dict[str, Any]:
254
  attempt_number=request.attempt_number,
255
  max_steps=request.max_steps,
256
  max_new_tokens=request.max_new_tokens,
257
- target_model=request.target_model,
258
  )
259
  except RuntimeError as exc:
260
  raise HTTPException(status_code=409, detail=str(exc)) from exc
 
99
  attempt_number: int = 1
100
  max_steps: int = 1
101
  max_new_tokens: int = 512
 
102
 
103
 
104
  def _metadata() -> dict[str, Any]:
 
253
  attempt_number=request.attempt_number,
254
  max_steps=request.max_steps,
255
  max_new_tokens=request.max_new_tokens,
 
256
  )
257
  except RuntimeError as exc:
258
  raise HTTPException(status_code=409, detail=str(exc)) from exc
server/runtime.py CHANGED
@@ -49,15 +49,6 @@ def _json_safe(value: Any) -> Any:
49
  return value
50
 
51
 
52
- def _safe_float(value: Any) -> float | None:
53
- try:
54
- if value is None:
55
- return None
56
- return float(value)
57
- except (TypeError, ValueError):
58
- return None
59
-
60
-
61
  @dataclass
62
  class ModelState:
63
  loaded: bool = False
@@ -233,28 +224,13 @@ class SpaceModelRegistry:
233
 
234
  def _base_generation_stack(self) -> tuple[Any, Any, dict[str, Any]]:
235
  try:
236
- self.load_base_model()
237
  except Exception as exc:
238
  raise RuntimeError(f"Base model load failed: {exc}") from exc
239
  with self._lock:
240
  if self._base_model is None or self._base_tokenizer is None:
241
  raise RuntimeError("Base model could not be loaded for fallback generation.")
242
- return self._base_model, self._base_tokenizer, self._base_status_payload()
243
-
244
- def _base_status_payload(self) -> dict[str, Any]:
245
- with self._lock:
246
- payload = {
247
- "loaded": self._base_model is not None and self._base_tokenizer is not None,
248
- "active_model_kind": "base",
249
- "source_repo_id": self._state.source_repo_id,
250
- "local_path": self._base_model_name(),
251
- "revision": None,
252
- "base_model_name": self._base_model_name(),
253
- "loaded_at": _iso_or_none(self._state.loaded_at),
254
- "error": None,
255
- "cache_dir": str(self.cache_dir),
256
- }
257
- return payload
258
 
259
  def _generate_with_possible_base_fallback(
260
  self,
@@ -293,26 +269,6 @@ class SpaceModelRegistry:
293
  return completion, fallback_state
294
  raise RuntimeError(f"Generation failed: {exc}") from exc
295
 
296
- def _generate_with_stack(
297
- self,
298
- *,
299
- prompt: str,
300
- max_new_tokens: int,
301
- model: Any,
302
- tokenizer: Any,
303
- model_state: dict[str, Any],
304
- ) -> tuple[str, dict[str, Any]]:
305
- try:
306
- completion = generate_completion(
307
- model=model,
308
- tokenizer=tokenizer,
309
- prompt=prompt,
310
- max_new_tokens=max_new_tokens,
311
- )
312
- except Exception as exc:
313
- raise RuntimeError(f"Generation failed: {exc}") from exc
314
- return completion, model_state
315
-
316
  def load_base_model(self) -> dict[str, Any]:
317
  torch, _, model_components = self._require_runtime_dependencies()
318
  AutoModelForCausalLM, AutoTokenizer = model_components
@@ -320,15 +276,12 @@ class SpaceModelRegistry:
320
 
321
  with self._lock:
322
  if self._base_model is not None and self._base_tokenizer is not None:
323
- if self._model is None or self._tokenizer is None:
324
- self._set_state(
325
- base_model_name=base_model_name,
326
- active_model_kind="base",
327
- loaded=True,
328
- error=None,
329
- )
330
- else:
331
- self._set_state(base_model_name=base_model_name, error=None)
332
  return self.status_payload()
333
 
334
  if torch.cuda.is_available() and torch.cuda.is_bf16_supported():
@@ -356,18 +309,15 @@ class SpaceModelRegistry:
356
  model.eval()
357
  self._base_model = model
358
  self._base_tokenizer = tokenizer
359
- if self._model is None or self._tokenizer is None:
360
- self._set_state(
361
- loaded=True,
362
- active_model_kind="base",
363
- base_model_name=base_model_name,
364
- local_path=base_model_name,
365
- revision=None,
366
- loaded_at=_utc_now(),
367
- error=None,
368
- )
369
- else:
370
- self._set_state(base_model_name=base_model_name, error=None)
371
  return self.status_payload()
372
 
373
  def load_from_local(
@@ -566,7 +516,6 @@ class SpaceModelRegistry:
566
  attempt_number: int = 1,
567
  max_steps: int = 1,
568
  max_new_tokens: int = 512,
569
- target_model: str = "current",
570
  ) -> dict[str, Any]:
571
  prompt = build_solver_prompt(
572
  {
@@ -581,30 +530,11 @@ class SpaceModelRegistry:
581
  "feedback": feedback or "No previous attempt yet. Solve the problem directly.",
582
  }
583
  )
584
- if target_model == "base":
585
- model, tokenizer, model_state = self._base_generation_stack()
586
- completion, model_state = self._generate_with_stack(
587
- prompt=prompt,
588
- max_new_tokens=max_new_tokens,
589
- model=model,
590
- tokenizer=tokenizer,
591
- model_state=model_state,
592
- )
593
- elif target_model == "trained":
594
- model, tokenizer, model_state = self._active_generation_stack(allow_base_fallback=False)
595
- completion, model_state = self._generate_with_stack(
596
- prompt=prompt,
597
- max_new_tokens=max_new_tokens,
598
- model=model,
599
- tokenizer=tokenizer,
600
- model_state=model_state,
601
- )
602
- else:
603
- completion, model_state = self._generate_with_possible_base_fallback(
604
- prompt=prompt,
605
- max_new_tokens=max_new_tokens,
606
- allow_base_fallback=True,
607
- )
608
  return {
609
  "problem_id": problem_id,
610
  "problem_type": problem_type,
@@ -614,8 +544,6 @@ class SpaceModelRegistry:
614
  "code": extract_code(completion),
615
  "model": model_state,
616
  "system_prompt": SYSTEM_PROMPT,
617
- "requested_model": target_model,
618
- "effective_model": model_state.get("active_model_kind", "unavailable"),
619
  }
620
 
621
 
@@ -713,95 +641,13 @@ class SpaceTrainingManager:
713
  )
714
  self._persist_status()
715
 
716
- def _read_json_file(self, path: str | Path | None) -> dict[str, Any] | None:
717
- if not path:
718
- return None
719
- candidate = Path(path)
720
- if not candidate.exists():
721
- return None
722
- try:
723
- payload = json.loads(candidate.read_text(encoding="utf-8"))
724
- except Exception:
725
- return None
726
- return payload if isinstance(payload, dict) else None
727
-
728
- def _find_latest_run_summary(self) -> dict[str, Any] | None:
729
- current_summary = self._read_json_file(self._job.run_summary_path)
730
- if current_summary is not None:
731
- return current_summary
732
-
733
- latest_summary: dict[str, Any] | None = None
734
- latest_mtime = -1.0
735
- for candidate in self.runs_dir.glob("**/logs/run_summary.json"):
736
- try:
737
- mtime = candidate.stat().st_mtime
738
- except OSError:
739
- continue
740
- if mtime <= latest_mtime:
741
- continue
742
- payload = self._read_json_file(candidate)
743
- if payload is None:
744
- continue
745
- latest_summary = payload
746
- latest_mtime = mtime
747
- return latest_summary
748
-
749
- def _demo_metrics_payload(self) -> dict[str, Any]:
750
- with self._lock:
751
- baseline_summary = dict(self._job.baseline_summary)
752
- trained_summary = dict(self._job.trained_summary)
753
- training_status = self._job.status
754
- phase = self._job.phase
755
- progress_ratio = float(self._job.progress_ratio or 0.0)
756
- completed_steps = int(self._job.completed_steps or 0)
757
- total_steps = int(self._job.total_steps or 0)
758
-
759
- run_summary = self._find_latest_run_summary() or {}
760
- final_metrics = run_summary.get("final_metrics", {}) if isinstance(run_summary.get("final_metrics"), dict) else {}
761
- rolling_metrics = (
762
- run_summary.get("rolling_metrics", {}) if isinstance(run_summary.get("rolling_metrics"), dict) else {}
763
- )
764
- final_trained_summary = (
765
- final_metrics.get("trained_summary", {}) if isinstance(final_metrics.get("trained_summary"), dict) else {}
766
- )
767
- final_baseline_summary = (
768
- final_metrics.get("baseline_summary", {}) if isinstance(final_metrics.get("baseline_summary"), dict) else {}
769
- )
770
-
771
- trained_overall = _safe_float(trained_summary.get("overall"))
772
- baseline_overall = _safe_float(baseline_summary.get("overall"))
773
- if trained_overall is None:
774
- trained_overall = _safe_float(final_trained_summary.get("overall"))
775
- if baseline_overall is None:
776
- baseline_overall = _safe_float(final_baseline_summary.get("overall"))
777
-
778
- live_pass_rate = _safe_float(rolling_metrics.get("avg_pass_rate"))
779
- overall_accuracy = trained_overall
780
- metric_source = "trained_eval" if overall_accuracy is not None else "unavailable"
781
- if overall_accuracy is None and live_pass_rate is not None:
782
- overall_accuracy = live_pass_rate
783
- metric_source = "rolling_pass_rate"
784
-
785
- return {
786
- "overall_accuracy": overall_accuracy,
787
- "baseline_accuracy": baseline_overall,
788
- "live_pass_rate": live_pass_rate,
789
- "training_status": training_status,
790
- "phase": phase,
791
- "progress_ratio": round(progress_ratio, 4),
792
- "completed_steps": completed_steps,
793
- "total_steps": total_steps,
794
- "metric_source": metric_source,
795
- }
796
-
797
  def status_payload(self) -> dict[str, Any]:
798
  with self._lock:
799
  payload = self._job.to_dict()
800
  payload["output_root"] = str(self.output_root)
801
  payload["status_file"] = str(self.status_file)
802
  payload["active"] = payload["status"] == "running"
803
- payload["demo_metrics"] = self._demo_metrics_payload()
804
- return payload
805
 
806
  def model_status_payload(self) -> dict[str, Any]:
807
  return self.model_registry.status_payload()
@@ -1048,7 +894,6 @@ class SpaceTrainingManager:
1048
  attempt_number: int = 1,
1049
  max_steps: int = 1,
1050
  max_new_tokens: int = 512,
1051
- target_model: str = "current",
1052
  ) -> dict[str, Any]:
1053
  return self.model_registry.generate_code(
1054
  problem=problem,
@@ -1061,5 +906,4 @@ class SpaceTrainingManager:
1061
  attempt_number=attempt_number,
1062
  max_steps=max_steps,
1063
  max_new_tokens=max_new_tokens,
1064
- target_model=target_model,
1065
  )
 
49
  return value
50
 
51
 
 
 
 
 
 
 
 
 
 
52
  @dataclass
53
  class ModelState:
54
  loaded: bool = False
 
224
 
225
  def _base_generation_stack(self) -> tuple[Any, Any, dict[str, Any]]:
226
  try:
227
+ base_state = self.load_base_model()
228
  except Exception as exc:
229
  raise RuntimeError(f"Base model load failed: {exc}") from exc
230
  with self._lock:
231
  if self._base_model is None or self._base_tokenizer is None:
232
  raise RuntimeError("Base model could not be loaded for fallback generation.")
233
+ return self._base_model, self._base_tokenizer, base_state
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
234
 
235
  def _generate_with_possible_base_fallback(
236
  self,
 
269
  return completion, fallback_state
270
  raise RuntimeError(f"Generation failed: {exc}") from exc
271
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
272
  def load_base_model(self) -> dict[str, Any]:
273
  torch, _, model_components = self._require_runtime_dependencies()
274
  AutoModelForCausalLM, AutoTokenizer = model_components
 
276
 
277
  with self._lock:
278
  if self._base_model is not None and self._base_tokenizer is not None:
279
+ self._set_state(
280
+ base_model_name=base_model_name,
281
+ active_model_kind="base",
282
+ loaded=True,
283
+ error=None,
284
+ )
 
 
 
285
  return self.status_payload()
286
 
287
  if torch.cuda.is_available() and torch.cuda.is_bf16_supported():
 
309
  model.eval()
310
  self._base_model = model
311
  self._base_tokenizer = tokenizer
312
+ self._set_state(
313
+ loaded=True,
314
+ active_model_kind="base",
315
+ base_model_name=base_model_name,
316
+ local_path=base_model_name,
317
+ revision=None,
318
+ loaded_at=_utc_now(),
319
+ error=None,
320
+ )
 
 
 
321
  return self.status_payload()
322
 
323
  def load_from_local(
 
516
  attempt_number: int = 1,
517
  max_steps: int = 1,
518
  max_new_tokens: int = 512,
 
519
  ) -> dict[str, Any]:
520
  prompt = build_solver_prompt(
521
  {
 
530
  "feedback": feedback or "No previous attempt yet. Solve the problem directly.",
531
  }
532
  )
533
+ completion, model_state = self._generate_with_possible_base_fallback(
534
+ prompt=prompt,
535
+ max_new_tokens=max_new_tokens,
536
+ allow_base_fallback=True,
537
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
538
  return {
539
  "problem_id": problem_id,
540
  "problem_type": problem_type,
 
544
  "code": extract_code(completion),
545
  "model": model_state,
546
  "system_prompt": SYSTEM_PROMPT,
 
 
547
  }
548
 
549
 
 
641
  )
642
  self._persist_status()
643
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
644
  def status_payload(self) -> dict[str, Any]:
645
  with self._lock:
646
  payload = self._job.to_dict()
647
  payload["output_root"] = str(self.output_root)
648
  payload["status_file"] = str(self.status_file)
649
  payload["active"] = payload["status"] == "running"
650
+ return payload
 
651
 
652
  def model_status_payload(self) -> dict[str, Any]:
653
  return self.model_registry.status_payload()
 
894
  attempt_number: int = 1,
895
  max_steps: int = 1,
896
  max_new_tokens: int = 512,
 
897
  ) -> dict[str, Any]:
898
  return self.model_registry.generate_code(
899
  problem=problem,
 
906
  attempt_number=attempt_number,
907
  max_steps=max_steps,
908
  max_new_tokens=max_new_tokens,
 
909
  )