""" Nova-1-XL Dataset Generation - HF Space UI By SmilyAI Labs """ import os import json import threading import gradio as gr HF_TOKEN = os.environ.get("HF_TOKEN", "") os.environ["HF_TOKEN"] = HF_TOKEN LOCAL_CACHE = "./nova_datagen_cache" os.makedirs(LOCAL_CACHE, exist_ok=True) from datagen import generate_dataset, HF_DATASET_REPO # ── Global state ─────────────────────────────────────────────────────────────── is_running = False gen_thread = None start_time = None # ── Status reader ────────────────────────────────────────────────────────────── def get_status() -> str: meta_path = os.path.join(LOCAL_CACHE, "metadata.json") if not os.path.exists(meta_path): if is_running: return ( "## 🔄 Starting up...\n\n" "Producers are launching. First sample coming soon.\n\n" f"Check logs for HTTP 200 responses from endpoint." ) return "## ⏸️ Ready\n\nClick **Start Generation** to begin." try: with open(meta_path) as f: d = json.load(f) status = d.get("status", "unknown") n = d.get("n_generated", 0) total_tok = d.get("total_tokens", 0) target_tok = d.get("target_tokens", 815_000_000) pct = d.get("pct_complete", 0) rate = d.get("rate_mh", 0) elapsed = d.get("elapsed_h", 0) eta = (target_tok - total_tok) / max(rate * 1_000_000, 1) / 3600 cats = d.get("cat_counts", {}) q_size = d.get("queue_size", 0) avg_r = d.get("avg_reasoning_len", 0) avg_a = d.get("avg_response_len", 0) dupes = d.get("duplicates", 0) fails = d.get("failures", 0) too_long = d.get("too_long", 0) icon = "✅" if status == "complete" else ("🔄" if is_running else "⏸️") # Category breakdown cat_lines = "\n".join( f"| {k} | {v:,} |" for k, v in sorted(cats.items(), key=lambda x: -x[1]) ) if cats else "| (none yet) | 0 |" return f""" ## {icon} Nova-1-XL Generation {'Complete!' if status == 'complete' else 'In Progress...' if is_running else 'Paused'} ### Progress | Metric | Value | |---|---| | **Samples Generated** | {n:,} | | **Tokens** | {total_tok/1e6:.1f}M / {target_tok/1e6:.0f}M | | **Complete** | {pct:.1f}% | | **Rate** | {rate:.1f}M tok/h | | **Elapsed** | {elapsed:.2f}h | | **ETA** | {eta:.1f}h | | **Queue Size** | {q_size} | ### Quality | Metric | Value | |---|---| | **Avg Reasoning Length** | {avg_r:,} chars | | **Avg Response Length** | {avg_a:,} chars | | **Duplicates Skipped** | {dupes:,} | | **Failures** | {fails:,} | | **Too Long (skipped)** | {too_long:,} | ### Category Breakdown | Category | Samples | |---|---| {cat_lines} ### Dataset [View on HF Hub](https://huggingface.co/datasets/{HF_DATASET_REPO}) """ except Exception as e: return f"## ⚠️ Error reading status\n\n```{e}```" def get_latest_sample() -> str: """Show the most recently generated sample.""" samples_dir = os.path.join(LOCAL_CACHE, "samples") if not os.path.exists(samples_dir): return "No samples yet." files = sorted(os.listdir(samples_dir)) if not files: return "No samples yet." latest = os.path.join(samples_dir, files[-1]) try: with open(latest, encoding="utf-8") as f: return f.read() except Exception as e: return f"Error reading sample: {e}" # ── Controls ─────────────────────────────────────────────────────────────────── def start_generation() -> str: global is_running, gen_thread, start_time if is_running: return "⚠️ Already running!" if gen_thread and gen_thread.is_alive(): return "⚠️ Thread still alive!" is_running = True import time start_time = time.time() def run(): global is_running try: generate_dataset() except Exception as e: import traceback print(f"Generation error: {e}") traceback.print_exc() finally: is_running = False gen_thread = threading.Thread(target=run, daemon=True, name="generator") gen_thread.start() return "🚀 Generation started! Status updates every 30 seconds." def stop_generation() -> str: global is_running is_running = False return "⏹️ Stop requested. Current sample will finish before stopping." def refresh() -> tuple: return get_status(), get_latest_sample() # ── UI ───────────────────────────────────────────────────────────────────────── with gr.Blocks( title="Nova-1-XL Dataset Generator", theme=gr.themes.Soft(), ) as demo: gr.Markdown(""" # 🌟 Nova-1-XL Dataset Generator **By SmilyAI Labs** | Generates personality-anchored reasoning training data """) with gr.Row(): start_btn = gr.Button("🚀 Start Generation", variant="primary", scale=3) stop_btn = gr.Button("⏹️ Stop", variant="stop", scale=1) refresh_btn = gr.Button("🔄 Refresh", variant="secondary", scale=1) msg_box = gr.Textbox( label="Control Message", interactive=False, max_lines=1, ) with gr.Row(): with gr.Column(scale=2): status_md = gr.Markdown(get_status()) with gr.Column(scale=3): sample_box = gr.Code( label="Latest Sample", language="markdown", lines=30, value=get_latest_sample(), ) # ── Auto-refresh timer ───────────────────────────────────────────────────── try: timer = gr.Timer(value=30) timer.tick(fn=refresh, outputs=[status_md, sample_box]) except AttributeError: gr.Markdown("*Auto-refresh unavailable - use Refresh button*") # ── Button actions ───────────────────────────────────────────────────────── start_btn.click( fn=start_generation, outputs=msg_box, ) stop_btn.click( fn=stop_generation, outputs=msg_box, ) refresh_btn.click( fn=refresh, outputs=[status_md, sample_box], ) if __name__ == "__main__": demo.launch( server_name="0.0.0.0", server_port=7860, )