Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| TIERS = { | |
| "brain": { | |
| "model": "deepseek-v4-pro", | |
| "accuracy": 88.4, | |
| "cost_per_query": 0.002, | |
| "description": "maximum quality, no shortcuts", | |
| }, | |
| "medium": { | |
| "model": "deepseek-v4-flash", | |
| "accuracy": 81.2, | |
| "cost_per_query": 0.000182, | |
| "description": "balanced speed and quality", | |
| }, | |
| "cheap": { | |
| "model": "varies (Gemini Flash, Llama, etc.)", | |
| "accuracy": 65.0, | |
| "cost_per_query": 0.0, | |
| "description": "free-tier models, zero cost", | |
| }, | |
| } | |
| BENCHMARKS = [ | |
| ["BigPickle OpenCoderPure", 117.7, 0.24, "5 free models propose, Brain refines split vote"], | |
| ["BigPickle FamilyDebate", 118.6, 0.30, "Google x Meta x Microsoft vote, Brain refines"], | |
| ["BigPickle MoA (3x Llama70B)", 120.6, 0.42, "3-pass Llama 70B multi-attention"], | |
| ["BigPickle FreeEnsemble", 110.9, 0.00, "9 free models majority vote (zero cost)"], | |
| ["BigPickle Opencodebate", 107.2, 0.60, "Flash / Llama70B debate, Brain breaks tie"], | |
| ["VibeUltraX (DeepSeek family)", 104.0, 0.46, "DeepSeek-only cascade + flash/pro debate"], | |
| ["Raw Brain (baseline)", 100.0, 1.00, "single deepseek-v4-pro baseline"], | |
| ["budget (trivial routing)", 40.0, 0.00, "direct routing to cheapest model"], | |
| ] | |
| def route_query(complexity, max_cost): | |
| if complexity < 3: | |
| tier = "cheap" | |
| elif complexity < 6: | |
| tier = "medium" | |
| else: | |
| tier = "brain" | |
| if max_cost < 0.25 and tier in ("medium", "brain"): | |
| tier = "cheap" | |
| selected = TIERS[tier] | |
| brain = TIERS["brain"] | |
| savings = ((brain["cost_per_query"] - selected["cost_per_query"]) / brain["cost_per_query"]) * 100 | |
| quality_ratio = (selected["accuracy"] / brain["accuracy"]) * 100 | |
| return ( | |
| tier.upper(), | |
| selected["model"], | |
| selected["description"], | |
| f"${selected['cost_per_query']:.6f}", | |
| f"{quality_ratio:.1f}% of brain quality", | |
| f"{savings:.0f}% cheaper than brain", | |
| ) | |
| with gr.Blocks( | |
| theme=gr.themes.Soft(primary_hue="indigo", neutral_hue="slate"), | |
| title="vibeOScore", | |
| ) as demo: | |
| gr.Markdown("# vibeOScore \u2014 Model Routing Simulator") | |
| gr.Markdown( | |
| "Routes AI prompts across model tiers based on complexity vs cost constraints. " | |
| "Brain (best quality) \u2192 Medium (balanced) \u2192 Cheap (free, zero cost)." | |
| ) | |
| with gr.Tab("Routing Simulator"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| complexity = gr.Slider(1, 10, value=5, step=1, label="Query complexity") | |
| max_cost = gr.Slider(0.0, 1.0, value=0.5, step=0.05, label="Max cost (fraction of brain tier)") | |
| gr.Examples( | |
| examples=[ | |
| [2, 0.0], | |
| [5, 0.3], | |
| [9, 0.8], | |
| ], | |
| inputs=[complexity, max_cost], | |
| label="Quick presets", | |
| ) | |
| route_btn = gr.Button("Route", variant="primary", size="sm") | |
| with gr.Column(): | |
| tier_out = gr.Textbox(label="Assigned tier") | |
| model_out = gr.Textbox(label="Model") | |
| desc_out = gr.Textbox(label="Why this tier") | |
| cost_out = gr.Textbox(label="Cost per query") | |
| quality_out = gr.Textbox(label="Quality vs brain") | |
| savings_out = gr.Textbox(label="Savings vs brain") | |
| route_btn.click( | |
| fn=route_query, | |
| inputs=[complexity, max_cost], | |
| outputs=[tier_out, model_out, desc_out, cost_out, quality_out, savings_out], | |
| ) | |
| with gr.Tab("Benchmarks"): | |
| gr.Markdown("**Benchmark results** \u2014 660k evaluations across 11 strategies x 30 runs x 2000 MC questions. Quality scores normalized to Raw Brain = 100%.") | |
| gr.Dataframe( | |
| headers=["Strategy", "Quality vs Brain", "Cost vs Brain", "Method"], | |
| value=[[s, f"{q:.1f}%", f"{c*100:.0f}%", m] for s, q, c, m in BENCHMARKS], | |
| interactive=False, | |
| column_widths=["220px", "130px", "130px", "auto"], | |
| ) | |
| gr.Markdown( | |
| "The FreeEnsemble (9 free models, zero cost) hits **110.9% quality at $0**. " | |
| "OpenCoderPure achieves **117.7% quality at 24% cost** \u2014 best Pareto efficiency." | |
| ) | |
| with gr.Tab("About"): | |
| gr.Markdown( | |
| "**vibeOScore** \u2014 open-source AI agent orchestration backend.\n\n" | |
| "Built with Fastify v5, TypeScript, SQLite.\n\n" | |
| "**Features:**\n" | |
| "- Model tier routing (brain / medium / cheap)\n" | |
| "- Multi-provider cost optimization\n" | |
| "- MCP server for agent-to-agent communication\n" | |
| "- LSH approximate caching (90.9% hit rate)\n" | |
| "- INT8 vector quantization (3-6x speedup)\n" | |
| "- SPI multi-resolution indexing\n\n" | |
| "**Links:** [GitHub](https://github.com/DrunkkToys/vibeOScore)" | |
| " | [npm](https://www.npmjs.com/package/vibeoscore)" | |
| " | [Frontend](https://github.com/DrunkkToys/theSaver-oc)" | |
| " | [MCP server](https://github.com/DrunkkToys/vibeOSmcp)" | |
| ) | |
| demo.launch() | |