| [ |
| { |
| "model_name": "VTXAI/vtx-embed-1M", |
| "model_label": "LF4 vtx-embed-1M (nano)", |
| "embedding_dim": 64, |
| "model_size_mb": 0.786432, |
| "index_dir": "/tmp/vtx_bench_LF4_vtx-embed-1M_(nano)_onbo80p2", |
| "indexed_files": 330, |
| "total_chunks": 2040, |
| "chunk_size": 1500, |
| "chunk_overlap": 200, |
| "languages": { |
| "python": 1839, |
| "typescript": 27, |
| "tsx": 129, |
| "javascript": 10, |
| "bash": 16, |
| "css": 13, |
| "powershell": 5 |
| }, |
| "index_time_ms": 16107.8, |
| "peak_ram_mb": 0.0, |
| "memo_hits": 0, |
| "memo_misses": 2040, |
| "query_results": [ |
| { |
| "query": "agent runner main loop execution", |
| "description": "Agent runner entry point", |
| "expected_file": "agent_runner", |
| "latency_ms": 5.53, |
| "top1_file": "Site/src/content/blog/agentic-loop.ts", |
| "top5_files": [ |
| "Site/src/content/blog/agentic-loop.ts", |
| "src/vtx/agent_runner.py", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/loop.py", |
| "benchmark.py" |
| ], |
| "top10_files": [ |
| "Site/src/content/blog/agentic-loop.ts", |
| "src/vtx/agent_runner.py", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/loop.py", |
| "benchmark.py", |
| "src/vtx/runtime.py", |
| "src/vtx/ui/agent_runner.py", |
| "src/vtx/sdk/handoffs.py", |
| "src/vtx/agents/loader.py", |
| "src/vtx/agents/registry.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0216 |
| }, |
| { |
| "query": "how does the agent run tasks asynchronously", |
| "description": "Async task execution", |
| "expected_file": "agent_runner", |
| "latency_ms": 4.28, |
| "top1_file": "src/vtx/runtime.py", |
| "top5_files": [ |
| "src/vtx/runtime.py", |
| "Site/src/content/blog/agentic-loop.ts", |
| "src/vtx/agents/loader.py", |
| "benchmark.py", |
| "src/vtx/tools/task.py" |
| ], |
| "top10_files": [ |
| "src/vtx/runtime.py", |
| "Site/src/content/blog/agentic-loop.ts", |
| "src/vtx/agents/loader.py", |
| "benchmark.py", |
| "src/vtx/tools/task.py", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/prompts/identity.py", |
| "src/vtx/loop.py", |
| "src/vtx/sdk/run_config.py", |
| "src/vtx/sdk/results.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0233 |
| }, |
| { |
| "query": "command line interface argument parsing", |
| "description": "CLI argument parsing", |
| "expected_file": "cli", |
| "latency_ms": 5.21, |
| "top1_file": "src/vtx/cli.py", |
| "top5_files": [ |
| "src/vtx/cli.py", |
| "src/vtx/tools/bash.py", |
| "benchmark.py", |
| "src/vtx/ui/agent_runner.py", |
| "src/vtx/ui/input.py" |
| ], |
| "top10_files": [ |
| "src/vtx/cli.py", |
| "src/vtx/tools/bash.py", |
| "benchmark.py", |
| "src/vtx/ui/agent_runner.py", |
| "src/vtx/ui/input.py", |
| "src/vtx/ui/formatting.py", |
| "src/vtx/ui/commands/sessions.py", |
| "src/vtx/llm/tool_parser.py", |
| "src/vtx/llm/sdk/openai.py", |
| "src/vtx/loop.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0172 |
| }, |
| { |
| "query": "vtx CLI entry point argparse", |
| "description": "CLI argparse setup", |
| "expected_file": "cli", |
| "latency_ms": 4.7, |
| "top1_file": "src/vtx/cli.py", |
| "top5_files": [ |
| "src/vtx/cli.py", |
| "benchmark.py", |
| "src/vtx/gh_cli.py", |
| "scripts/show_themes.py", |
| "src/vtx/extensions.py" |
| ], |
| "top10_files": [ |
| "src/vtx/cli.py", |
| "benchmark.py", |
| "src/vtx/gh_cli.py", |
| "scripts/show_themes.py", |
| "src/vtx/extensions.py", |
| "src/vtx/session.py", |
| "src/vtx/llm/provider_catalog.py", |
| "src/vtx/turn.py", |
| "src/vtx/diff_display.py", |
| "src/vtx/runtime.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.025 |
| }, |
| { |
| "query": "configuration loading and defaults", |
| "description": "Config loading", |
| "expected_file": "config", |
| "latency_ms": 4.58, |
| "top1_file": "src/vtx/config.py", |
| "top5_files": [ |
| "src/vtx/config.py", |
| "src/vtx/llm/oauth/dynamic.py", |
| "src/vtx/llm/provider_catalog.py", |
| "src/vtx/sdk/run_config.py", |
| "src/vtx/llm/sdk/supercode.py" |
| ], |
| "top10_files": [ |
| "src/vtx/config.py", |
| "src/vtx/llm/oauth/dynamic.py", |
| "src/vtx/llm/provider_catalog.py", |
| "src/vtx/sdk/run_config.py", |
| "src/vtx/llm/sdk/supercode.py", |
| "benchmark.py", |
| "src/vtx/sdk/agent.py", |
| "src/vtx/llm/dynamic_models.py", |
| "Site/src/content/docs/index.ts", |
| "src/vtx/runtime.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0202 |
| }, |
| { |
| "query": "user config file path resolution", |
| "description": "Config path resolution", |
| "expected_file": "config", |
| "latency_ms": 4.56, |
| "top1_file": "src/vtx/config.py", |
| "top5_files": [ |
| "src/vtx/config.py", |
| "src/vtx/extensions.py", |
| "scripts/show_themes.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/ui/path_complete.py" |
| ], |
| "top10_files": [ |
| "src/vtx/config.py", |
| "src/vtx/extensions.py", |
| "scripts/show_themes.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/ui/path_complete.py", |
| "src/vtx/tools/edit.py", |
| "benchmark.py", |
| "src/vtx/tools/skill.py", |
| "src/vtx/llm/provider_catalog.py", |
| "src/vtx/context/agent_mds.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0198 |
| }, |
| { |
| "query": "LLM API call streaming response", |
| "description": "LLM streaming", |
| "expected_file": "llm", |
| "latency_ms": 4.74, |
| "top1_file": "src/vtx/ui/blocks.py", |
| "top5_files": [ |
| "src/vtx/ui/blocks.py", |
| "src/vtx/llm/sdk/supercode.py", |
| "src/vtx/turn.py", |
| "src/vtx/llm/phase_parser.py", |
| "src/vtx/llm/base.py" |
| ], |
| "top10_files": [ |
| "src/vtx/ui/blocks.py", |
| "src/vtx/llm/sdk/supercode.py", |
| "src/vtx/turn.py", |
| "src/vtx/llm/phase_parser.py", |
| "src/vtx/llm/base.py", |
| "src/vtx/loop.py", |
| "benchmark.py", |
| "src/vtx/core/compaction.py", |
| "src/vtx/llm/providers/mock.py", |
| "src/vtx/llm/dynamic_models.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.016 |
| }, |
| { |
| "query": "model provider selection gemini openai", |
| "description": "Model provider", |
| "expected_file": "llm", |
| "latency_ms": 4.37, |
| "top1_file": "src/vtx/llm/providers/openai_sdk.py", |
| "top5_files": [ |
| "src/vtx/llm/providers/openai_sdk.py", |
| "src/vtx/runtime.py", |
| "src/vtx/ui/commands/providers.py", |
| "src/vtx/llm/models.py", |
| "src/vtx/ui/commands/auth.py" |
| ], |
| "top10_files": [ |
| "src/vtx/llm/providers/openai_sdk.py", |
| "src/vtx/runtime.py", |
| "src/vtx/ui/commands/providers.py", |
| "src/vtx/llm/models.py", |
| "src/vtx/ui/commands/auth.py", |
| "src/vtx/ui/selection_mode.py", |
| "src/vtx/llm/provider_catalog.py", |
| "src/vtx/sdk/agent.py", |
| "src/vtx/ui/completion_ui.py", |
| "src/vtx/llm/model_fetcher.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0192 |
| }, |
| { |
| "query": "token counting and context window management", |
| "description": "Token counting", |
| "expected_file": "llm", |
| "latency_ms": 4.62, |
| "top1_file": "src/vtx/core/compaction.py", |
| "top5_files": [ |
| "src/vtx/core/compaction.py", |
| "src/vtx/ui/widgets.py", |
| "Site/src/content/blog/memory-compaction.ts", |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/session.py" |
| ], |
| "top10_files": [ |
| "src/vtx/core/compaction.py", |
| "src/vtx/ui/widgets.py", |
| "Site/src/content/blog/memory-compaction.ts", |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/session.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/sdk/tracing/tracing_impl.py", |
| "src/vtx/llm/context_length.py", |
| "benchmark.py", |
| "src/vtx/context_governance.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0152 |
| }, |
| { |
| "query": "context window governance truncation", |
| "description": "Context governance", |
| "expected_file": "context_governance", |
| "latency_ms": 5.18, |
| "top1_file": "src/vtx/context_governance.py", |
| "top5_files": [ |
| "src/vtx/context_governance.py", |
| "src/vtx/core/compaction.py", |
| "benchmark.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/context/agent_mds.py" |
| ], |
| "top10_files": [ |
| "src/vtx/context_governance.py", |
| "src/vtx/core/compaction.py", |
| "benchmark.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/context/agent_mds.py", |
| "Site/src/content/blog/memory-compaction.ts", |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/ui/widgets.py", |
| "src/vtx/runtime.py", |
| "src/vtx/loop.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0182 |
| }, |
| { |
| "query": "context item priority ranking", |
| "description": "Context priority", |
| "expected_file": "context", |
| "latency_ms": 4.5, |
| "top1_file": "src/vtx/ui/input.py", |
| "top5_files": [ |
| "src/vtx/ui/input.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/ui/tree.py", |
| "src/vtx/sdk/items.py", |
| "src/vtx/ui/completion_ui.py" |
| ], |
| "top10_files": [ |
| "src/vtx/ui/input.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/ui/tree.py", |
| "src/vtx/sdk/items.py", |
| "src/vtx/ui/completion_ui.py", |
| "src/vtx/context/skills.py", |
| "src/vtx/context/git.py", |
| "src/vtx/sdk/items_base.py", |
| "src/vtx/llm/context_length.py", |
| "Site/src/index.css" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": true, |
| "top1_score": 0.0182 |
| }, |
| { |
| "query": "tool dispatcher routing function calls", |
| "description": "Tool dispatcher", |
| "expected_file": "dispatcher", |
| "latency_ms": 5.9, |
| "top1_file": "src/vtx/dispatcher.py", |
| "top5_files": [ |
| "src/vtx/dispatcher.py", |
| "benchmark.py", |
| "src/vtx/llm/tool_parser.py", |
| "src/vtx/context_governance.py", |
| "src/vtx/llm/sdk/supercode.py" |
| ], |
| "top10_files": [ |
| "src/vtx/dispatcher.py", |
| "benchmark.py", |
| "src/vtx/llm/tool_parser.py", |
| "src/vtx/context_governance.py", |
| "src/vtx/llm/sdk/supercode.py", |
| "src/vtx/llm/providers/supercode.py", |
| "src/vtx/prompts/tooling.py", |
| "src/vtx/tools/read.py", |
| "src/vtx/sdk/tools.py", |
| "src/vtx/turn.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0187 |
| }, |
| { |
| "query": "dispatch tool call with arguments", |
| "description": "Tool call dispatch", |
| "expected_file": "dispatcher", |
| "latency_ms": 6.33, |
| "top1_file": "src/vtx/turn.py", |
| "top5_files": [ |
| "src/vtx/turn.py", |
| "benchmark.py", |
| "src/vtx/ui/session_ui.py", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/llm/tool_parser.py" |
| ], |
| "top10_files": [ |
| "src/vtx/turn.py", |
| "benchmark.py", |
| "src/vtx/ui/session_ui.py", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/llm/tool_parser.py", |
| "src/vtx/ui/tree.py", |
| "src/vtx/prompts/tooling.py", |
| "src/vtx/tools/base.py", |
| "src/vtx/tools/find.py", |
| "src/vtx/tools/task.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0188 |
| }, |
| { |
| "query": "event system publish subscribe hooks", |
| "description": "Event system", |
| "expected_file": "events", |
| "latency_ms": 6.25, |
| "top1_file": "src/vtx/hooks/loader.py", |
| "top5_files": [ |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/bridge.py", |
| "benchmark.py", |
| "src/vtx/hooks/agent_hook.py", |
| "src/vtx/hooks/registry.py" |
| ], |
| "top10_files": [ |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/bridge.py", |
| "benchmark.py", |
| "src/vtx/hooks/agent_hook.py", |
| "src/vtx/hooks/registry.py", |
| "src/vtx/loop.py", |
| "src/vtx/extensions.py", |
| "src/vtx/hooks/runtime.py", |
| "Site/src/hooks/useRouteMeta.ts", |
| "src/vtx/hooks/types.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0211 |
| }, |
| { |
| "query": "streaming event output to terminal", |
| "description": "Event streaming", |
| "expected_file": "events", |
| "latency_ms": 4.75, |
| "top1_file": "src/vtx/turn.py", |
| "top5_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/ui/blocks.py", |
| "benchmark.py", |
| "Site/src/components/FeaturesPage.tsx" |
| ], |
| "top10_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/ui/blocks.py", |
| "benchmark.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/sdk/guardrails/types.py", |
| "src/vtx/tools/task.py", |
| "src/vtx/events.py", |
| "Site/src/components/TerminalBlock.tsx", |
| "src/vtx/ui/agent_runner.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": true, |
| "top1_score": 0.0192 |
| }, |
| { |
| "query": "git branch creation and switching", |
| "description": "Git branch", |
| "expected_file": "git_branch", |
| "latency_ms": 4.5, |
| "top1_file": "src/vtx/git_branch.py", |
| "top5_files": [ |
| "src/vtx/git_branch.py", |
| "src/vtx/context/git.py", |
| "src/vtx/ui/widgets.py", |
| "benchmark.py", |
| "Site/src/components/Capabilities.tsx" |
| ], |
| "top10_files": [ |
| "src/vtx/git_branch.py", |
| "src/vtx/context/git.py", |
| "src/vtx/ui/widgets.py", |
| "benchmark.py", |
| "Site/src/components/Capabilities.tsx", |
| "src/vtx/ui/startup.py", |
| "scripts/install.sh", |
| "src/vtx/ui/commands/settings.py", |
| "Site/src/content/blog/agentskill.ts", |
| "src/vtx/config.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0268 |
| }, |
| { |
| "query": "github CLI integration shell command", |
| "description": "GitHub CLI", |
| "expected_file": "gh_cli", |
| "latency_ms": 4.53, |
| "top1_file": "src/vtx/ui/agent_runner.py", |
| "top5_files": [ |
| "src/vtx/ui/agent_runner.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/tools/bash.py", |
| "scripts/install.ps1", |
| "scripts/install.sh" |
| ], |
| "top10_files": [ |
| "src/vtx/ui/agent_runner.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/tools/bash.py", |
| "scripts/install.ps1", |
| "scripts/install.sh", |
| "Site/src/hooks/useRouteMeta.ts", |
| "benchmark.py", |
| "src/vtx/ui/commands/auth.py", |
| "src/vtx/cli.py", |
| "tests/ui/test_shell_command_detection.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0174 |
| }, |
| { |
| "query": "diff display unified patch format", |
| "description": "Diff display", |
| "expected_file": "diff_display", |
| "latency_ms": 5.29, |
| "top1_file": "src/vtx/tools/edit.py", |
| "top5_files": [ |
| "src/vtx/tools/edit.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/ui/blocks.py", |
| "benchmark.py", |
| "src/vtx/diff_display.py" |
| ], |
| "top10_files": [ |
| "src/vtx/tools/edit.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/ui/blocks.py", |
| "benchmark.py", |
| "src/vtx/diff_display.py", |
| "src/vtx/ui/widgets.py", |
| "src/vtx/ui/formatting.py", |
| "Site/src/index.css", |
| "src/vtx/tools/task.py", |
| "Site/src/components/MarkdownRenderer.tsx" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0166 |
| }, |
| { |
| "query": "show file changes colored diff output", |
| "description": "Colored diff", |
| "expected_file": "diff_display", |
| "latency_ms": 5.76, |
| "top1_file": "src/vtx/ui/widgets.py", |
| "top5_files": [ |
| "src/vtx/ui/widgets.py", |
| "src/vtx/ui/launch.py", |
| "src/vtx/ui/blocks.py", |
| "src/vtx/tools/edit.py", |
| "src/vtx/ui/chat.py" |
| ], |
| "top10_files": [ |
| "src/vtx/ui/widgets.py", |
| "src/vtx/ui/launch.py", |
| "src/vtx/ui/blocks.py", |
| "src/vtx/tools/edit.py", |
| "src/vtx/ui/chat.py", |
| "benchmark.py", |
| "src/vtx/tools/write.py", |
| "Site/src/index.css", |
| "src/vtx/ui/styles.py", |
| "src/vtx/ui/commands/settings.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0191 |
| }, |
| { |
| "query": "plugin extension registration loading", |
| "description": "Extension loading", |
| "expected_file": "extensions", |
| "latency_ms": 5.5, |
| "top1_file": "src/vtx/extensions.py", |
| "top5_files": [ |
| "src/vtx/extensions.py", |
| "Site/src/content/blog/mcp-extensions.ts", |
| "benchmark.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/extensions.py" |
| ], |
| "top10_files": [ |
| "src/vtx/extensions.py", |
| "Site/src/content/blog/mcp-extensions.ts", |
| "benchmark.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/extensions.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "Site/scripts/prerender.mjs", |
| "Site/src/content/blog/mcp-extensions.ts", |
| "src/vtx/cli.py", |
| "src/vtx/llm/providers/supercode.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0229 |
| }, |
| { |
| "query": "builtin skill definition and registration", |
| "description": "Builtin skills", |
| "expected_file": "builtin_skill", |
| "latency_ms": 4.26, |
| "top1_file": "src/vtx/context/skills.py", |
| "top5_files": [ |
| "src/vtx/context/skills.py", |
| "src/vtx/tools/skill.py", |
| "src/vtx/sdk/skills.py", |
| "benchmark.py", |
| "Site/src/content/blog/agentskill.ts" |
| ], |
| "top10_files": [ |
| "src/vtx/context/skills.py", |
| "src/vtx/tools/skill.py", |
| "src/vtx/sdk/skills.py", |
| "benchmark.py", |
| "Site/src/content/blog/agentskill.ts", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/context/loader.py", |
| "src/vtx/context/skills.py", |
| "src/vtx/extensions.py", |
| "src/vtx/tools/skill.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0209 |
| }, |
| { |
| "query": "skill slash command frontmatter parsing", |
| "description": "Skill frontmatter", |
| "expected_file": "builtin_skill", |
| "latency_ms": 4.37, |
| "top1_file": "src/vtx/context/skills.py", |
| "top5_files": [ |
| "src/vtx/context/skills.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/autocomplete.py", |
| "src/vtx/ui/app.py", |
| "Site/src/content/blog/agentskill.ts" |
| ], |
| "top10_files": [ |
| "src/vtx/context/skills.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/autocomplete.py", |
| "src/vtx/ui/app.py", |
| "Site/src/content/blog/agentskill.ts", |
| "Site/src/components/Capabilities.tsx", |
| "src/vtx/tools/skill.py", |
| "src/vtx/context/skills.py", |
| "src/vtx/sdk/skills.py", |
| "src/vtx/ui/commands/agents.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0206 |
| }, |
| { |
| "query": "subagent invocation and communication", |
| "description": "Subagent invocation", |
| "expected_file": "agents", |
| "latency_ms": 4.08, |
| "top1_file": "src/vtx/tools/task.py", |
| "top5_files": [ |
| "src/vtx/tools/task.py", |
| "src/vtx/loop.py", |
| "src/vtx/config.py", |
| "src/vtx/ui/app.py", |
| "benchmark.py" |
| ], |
| "top10_files": [ |
| "src/vtx/tools/task.py", |
| "src/vtx/loop.py", |
| "src/vtx/config.py", |
| "src/vtx/ui/app.py", |
| "benchmark.py", |
| "src/vtx/llm/providers/openai_sdk.py", |
| "src/vtx/tools/background.py", |
| "src/vtx/tools/ask_user.py", |
| "src/vtx/ui/chat.py", |
| "src/vtx/agents/api.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": true, |
| "top1_score": 0.0169 |
| }, |
| { |
| "query": "agent message passing protocol", |
| "description": "Agent messaging", |
| "expected_file": "agents", |
| "latency_ms": 4.67, |
| "top1_file": "src/vtx/agents/loader.py", |
| "top5_files": [ |
| "src/vtx/agents/loader.py", |
| "src/vtx/ui/app_protocol.py", |
| "src/vtx/ui/commands/sessions.py", |
| "src/vtx/ui/commands/agents.py", |
| "src/vtx/agents/api.py" |
| ], |
| "top10_files": [ |
| "src/vtx/agents/loader.py", |
| "src/vtx/ui/app_protocol.py", |
| "src/vtx/ui/commands/sessions.py", |
| "src/vtx/ui/commands/agents.py", |
| "src/vtx/agents/api.py", |
| "src/vtx/agent_runner.py", |
| "src/vtx/runtime.py", |
| "src/vtx/hooks/agent_hook.py", |
| "src/vtx/ui/agent_runner.py", |
| "benchmark.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0127 |
| }, |
| { |
| "query": "pre-commit hook integration", |
| "description": "Pre-commit hooks", |
| "expected_file": "hooks", |
| "latency_ms": 4.65, |
| "top1_file": "src/vtx/hooks/loader.py", |
| "top5_files": [ |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/bridge.py", |
| "src/vtx/hooks/types.py", |
| "benchmark.py", |
| "src/vtx/hooks/agent_hook.py" |
| ], |
| "top10_files": [ |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/bridge.py", |
| "src/vtx/hooks/types.py", |
| "benchmark.py", |
| "src/vtx/hooks/agent_hook.py", |
| "src/vtx/hooks/registry.py", |
| "Site/scripts/prerender.mjs", |
| "src/vtx/hooks/__init__.py", |
| "src/vtx/hooks/runtime.py", |
| "src/vtx/hooks/bridge.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0214 |
| }, |
| { |
| "query": "async utility helpers gather timeout", |
| "description": "Async utilities", |
| "expected_file": "async_utils", |
| "latency_ms": 4.27, |
| "top1_file": "src/vtx/turn.py", |
| "top5_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/hooks/bridge.py", |
| "benchmark.py", |
| "src/vtx/tools/web.py", |
| "src/vtx/update_check.py" |
| ], |
| "top10_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/hooks/bridge.py", |
| "benchmark.py", |
| "src/vtx/tools/web.py", |
| "src/vtx/update_check.py", |
| "src/vtx/tools/background.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/llm/oauth/copilot.py", |
| "src/vtx/llm/sdk/anthropic.py", |
| "src/vtx/llm/sdk/openai.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.018 |
| }, |
| { |
| "query": "headless non-interactive execution mode", |
| "description": "Headless mode", |
| "expected_file": "headless", |
| "latency_ms": 4.91, |
| "top1_file": "Site/src/components/FeaturesPage.tsx", |
| "top5_files": [ |
| "Site/src/components/FeaturesPage.tsx", |
| "Site/src/hooks/useRouteMeta.ts", |
| "scripts/install.sh", |
| "src/vtx/headless.py", |
| "benchmark.py" |
| ], |
| "top10_files": [ |
| "Site/src/components/FeaturesPage.tsx", |
| "Site/src/hooks/useRouteMeta.ts", |
| "scripts/install.sh", |
| "src/vtx/headless.py", |
| "benchmark.py", |
| "Site/src/components/Capabilities.tsx", |
| ".agents/skills/vtx-tmux-test/run-e2e-tests.sh", |
| "src/vtx/agents/registry.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/ui/input.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0195 |
| }, |
| { |
| "query": "error handling exception retry logic", |
| "description": "Generic: error handling (open)", |
| "expected_file": null, |
| "latency_ms": 4.43, |
| "top1_file": "src/vtx/turn.py", |
| "top5_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/core/errors.py", |
| "src/vtx/llm/rate_limit.py", |
| "src/vtx/tools/background.py", |
| "benchmark.py" |
| ], |
| "top10_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/core/errors.py", |
| "src/vtx/llm/rate_limit.py", |
| "src/vtx/tools/background.py", |
| "benchmark.py", |
| "src/vtx/llm/sdk/openai.py", |
| "src/vtx/llm/sdk/anthropic.py", |
| "src/vtx/llm/base.py", |
| "src/vtx/sdk/permissions.py", |
| "src/vtx/extensions.py" |
| ], |
| "hit_at_1": null, |
| "hit_at_5": null, |
| "hit_at_10": null, |
| "top1_score": 0.018 |
| }, |
| { |
| "query": "logging setup structured log output", |
| "description": "Generic: logging (open)", |
| "expected_file": null, |
| "latency_ms": 4.74, |
| "top1_file": "src/vtx/loop.py", |
| "top5_files": [ |
| "src/vtx/loop.py", |
| "Site/src/content/blog/memory-compaction.ts", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/session_ui.py", |
| ".agents/skills/vtx-tmux-test/setup-test-project.sh" |
| ], |
| "top10_files": [ |
| "src/vtx/loop.py", |
| "Site/src/content/blog/memory-compaction.ts", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/session_ui.py", |
| ".agents/skills/vtx-tmux-test/setup-test-project.sh", |
| "src/vtx/llm/sdk/openai.py", |
| "benchmark.py", |
| "src/vtx/agents/api.py", |
| "src/vtx/ui/export.py", |
| "src/vtx/hooks/loader.py" |
| ], |
| "hit_at_1": null, |
| "hit_at_5": null, |
| "hit_at_10": null, |
| "top1_score": 0.0166 |
| }, |
| { |
| "query": "test fixtures mock patching pytest", |
| "description": "Test utilities", |
| "expected_file": "tests", |
| "latency_ms": 4.19, |
| "top1_file": "benchmark.py", |
| "top5_files": [ |
| "benchmark.py", |
| ".agents/skills/vtx-tmux-test/run-e2e-tests.sh", |
| "tests/llm/test_mock_provider.py", |
| "tests/ui/test_shell_command_detection.py", |
| "src/vtx/llm/dynamic_models.py" |
| ], |
| "top10_files": [ |
| "benchmark.py", |
| ".agents/skills/vtx-tmux-test/run-e2e-tests.sh", |
| "tests/llm/test_mock_provider.py", |
| "tests/ui/test_shell_command_detection.py", |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/llm/sdk/openai.py", |
| "Site/src/index.css", |
| "tests/sdk/test_provider_field.py", |
| "tests/test_agentic_loop.py", |
| "tests/sdk/test_integration.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0141 |
| } |
| ], |
| "recall_at_1": 0.3929, |
| "recall_at_5": 0.6071, |
| "recall_at_10": 0.7143, |
| "mean_latency_ms": 4.85, |
| "p50_latency_ms": 4.66, |
| "p95_latency_ms": 5.9, |
| "p99_latency_ms": 6.25, |
| "evaluated_queries": 28, |
| "open_queries": 2 |
| }, |
| { |
| "model_name": "VTXAI/vtx-embed-7M", |
| "model_label": "LF4 vtx-embed-7M (mini)", |
| "embedding_dim": 256, |
| "model_size_mb": 4.72448, |
| "index_dir": "/tmp/vtx_bench_LF4_vtx-embed-7M_(mini)_dt5m3366", |
| "indexed_files": 330, |
| "total_chunks": 2040, |
| "chunk_size": 1500, |
| "chunk_overlap": 200, |
| "languages": { |
| "python": 1839, |
| "typescript": 27, |
| "tsx": 129, |
| "javascript": 10, |
| "bash": 16, |
| "css": 13, |
| "powershell": 5 |
| }, |
| "index_time_ms": 3056.6, |
| "peak_ram_mb": 0.0, |
| "memo_hits": 0, |
| "memo_misses": 2040, |
| "query_results": [ |
| { |
| "query": "agent runner main loop execution", |
| "description": "Agent runner entry point", |
| "expected_file": "agent_runner", |
| "latency_ms": 6.35, |
| "top1_file": "Site/src/content/blog/agentic-loop.ts", |
| "top5_files": [ |
| "Site/src/content/blog/agentic-loop.ts", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/loop.py", |
| "src/vtx/agent_runner.py", |
| "src/vtx/runtime.py" |
| ], |
| "top10_files": [ |
| "Site/src/content/blog/agentic-loop.ts", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/loop.py", |
| "src/vtx/agent_runner.py", |
| "src/vtx/runtime.py", |
| "src/vtx/ui/agent_runner.py", |
| "src/vtx/sdk/handoffs.py", |
| "src/vtx/agents/loader.py", |
| "src/vtx/ui/commands/agents.py", |
| "src/vtx/sdk/run_config.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0228 |
| }, |
| { |
| "query": "how does the agent run tasks asynchronously", |
| "description": "Async task execution", |
| "expected_file": "agent_runner", |
| "latency_ms": 6.0, |
| "top1_file": "src/vtx/runtime.py", |
| "top5_files": [ |
| "src/vtx/runtime.py", |
| "Site/src/content/blog/agentic-loop.ts", |
| "src/vtx/agents/loader.py", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/tools/task.py" |
| ], |
| "top10_files": [ |
| "src/vtx/runtime.py", |
| "Site/src/content/blog/agentic-loop.ts", |
| "src/vtx/agents/loader.py", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/tools/task.py", |
| "src/vtx/tools/background.py", |
| "src/vtx/sdk/run_config.py", |
| "Site/src/content/blog/agentskill.ts", |
| "src/vtx/ui/commands/agents.py", |
| "src/vtx/sdk/results.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.024 |
| }, |
| { |
| "query": "command line interface argument parsing", |
| "description": "CLI argument parsing", |
| "expected_file": "cli", |
| "latency_ms": 5.96, |
| "top1_file": "src/vtx/cli.py", |
| "top5_files": [ |
| "src/vtx/cli.py", |
| "benchmark.py", |
| "scripts/show_themes.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/extensions.py" |
| ], |
| "top10_files": [ |
| "src/vtx/cli.py", |
| "benchmark.py", |
| "scripts/show_themes.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/extensions.py", |
| "src/vtx/ui/agent_runner.py", |
| "src/vtx/agents/api.py", |
| "src/vtx/tools/edit.py", |
| "src/vtx/ui/input.py", |
| "src/vtx/tools/find.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0195 |
| }, |
| { |
| "query": "vtx CLI entry point argparse", |
| "description": "CLI argparse setup", |
| "expected_file": "cli", |
| "latency_ms": 6.15, |
| "top1_file": "src/vtx/cli.py", |
| "top5_files": [ |
| "src/vtx/cli.py", |
| "src/vtx/extensions.py", |
| "scripts/show_themes.py", |
| "benchmark.py", |
| "src/vtx/gh_cli.py" |
| ], |
| "top10_files": [ |
| "src/vtx/cli.py", |
| "src/vtx/extensions.py", |
| "scripts/show_themes.py", |
| "benchmark.py", |
| "src/vtx/gh_cli.py", |
| "src/vtx/prompts/builder.py", |
| "src/vtx/ui/launch.py", |
| "src/vtx/session.py", |
| "src/vtx/cli.py", |
| "src/vtx/config.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0269 |
| }, |
| { |
| "query": "configuration loading and defaults", |
| "description": "Config loading", |
| "expected_file": "config", |
| "latency_ms": 6.13, |
| "top1_file": "src/vtx/config.py", |
| "top5_files": [ |
| "src/vtx/config.py", |
| "src/vtx/hooks/loader.py", |
| "src/vtx/llm/oauth/dynamic.py", |
| "src/vtx/sdk/run_config.py", |
| "src/vtx/llm/provider_catalog.py" |
| ], |
| "top10_files": [ |
| "src/vtx/config.py", |
| "src/vtx/hooks/loader.py", |
| "src/vtx/llm/oauth/dynamic.py", |
| "src/vtx/sdk/run_config.py", |
| "src/vtx/llm/provider_catalog.py", |
| "src/vtx/extensions.py", |
| "src/vtx/config.py", |
| "benchmark.py", |
| "Site/src/content/docs/index.ts", |
| "src/vtx/dispatcher.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0209 |
| }, |
| { |
| "query": "user config file path resolution", |
| "description": "Config path resolution", |
| "expected_file": "config", |
| "latency_ms": 6.22, |
| "top1_file": "src/vtx/config.py", |
| "top5_files": [ |
| "src/vtx/config.py", |
| "src/vtx/extensions.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/llm/provider_catalog.py", |
| "scripts/show_themes.py" |
| ], |
| "top10_files": [ |
| "src/vtx/config.py", |
| "src/vtx/extensions.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/llm/provider_catalog.py", |
| "scripts/show_themes.py", |
| "src/vtx/tools/edit.py", |
| "src/vtx/context/agent_mds.py", |
| "src/vtx/tools_manager.py", |
| "src/vtx/ui/path_complete.py", |
| "Site/vite.config.ts" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0201 |
| }, |
| { |
| "query": "LLM API call streaming response", |
| "description": "LLM streaming", |
| "expected_file": "llm", |
| "latency_ms": 5.86, |
| "top1_file": "src/vtx/llm/sdk/supercode.py", |
| "top5_files": [ |
| "src/vtx/llm/sdk/supercode.py", |
| "src/vtx/turn.py", |
| "src/vtx/ui/blocks.py", |
| "src/vtx/llm/providers/mock.py", |
| "src/vtx/llm/dynamic_models.py" |
| ], |
| "top10_files": [ |
| "src/vtx/llm/sdk/supercode.py", |
| "src/vtx/turn.py", |
| "src/vtx/ui/blocks.py", |
| "src/vtx/llm/providers/mock.py", |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/llm/phase_parser.py", |
| "src/vtx/llm/sdk/base.py", |
| "src/vtx/llm/base.py", |
| "src/vtx/llm/rate_limit.py", |
| "benchmark.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.018 |
| }, |
| { |
| "query": "model provider selection gemini openai", |
| "description": "Model provider", |
| "expected_file": "llm", |
| "latency_ms": 5.62, |
| "top1_file": "src/vtx/llm/providers/openai_sdk.py", |
| "top5_files": [ |
| "src/vtx/llm/providers/openai_sdk.py", |
| "src/vtx/ui/commands/providers.py", |
| "src/vtx/runtime.py", |
| "src/vtx/llm/sdk/openai.py", |
| "src/vtx/ui/commands/models.py" |
| ], |
| "top10_files": [ |
| "src/vtx/llm/providers/openai_sdk.py", |
| "src/vtx/ui/commands/providers.py", |
| "src/vtx/runtime.py", |
| "src/vtx/llm/sdk/openai.py", |
| "src/vtx/ui/commands/models.py", |
| "src/vtx/llm/models.py", |
| "src/vtx/ui/commands/auth.py", |
| "src/vtx/ui/selection_mode.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/app.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0182 |
| }, |
| { |
| "query": "token counting and context window management", |
| "description": "Token counting", |
| "expected_file": "llm", |
| "latency_ms": 5.65, |
| "top1_file": "src/vtx/llm/dynamic_models.py", |
| "top5_files": [ |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/ui/widgets.py", |
| "src/vtx/core/compaction.py", |
| "src/vtx/session.py", |
| "Site/src/content/blog/memory-compaction.ts" |
| ], |
| "top10_files": [ |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/ui/widgets.py", |
| "src/vtx/core/compaction.py", |
| "src/vtx/session.py", |
| "Site/src/content/blog/memory-compaction.ts", |
| "src/vtx/loop.py", |
| "src/vtx/tools/background.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/llm/model_fetcher.py", |
| "src/vtx/sdk/tracing/tracing_impl.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0175 |
| }, |
| { |
| "query": "context window governance truncation", |
| "description": "Context governance", |
| "expected_file": "context_governance", |
| "latency_ms": 6.55, |
| "top1_file": "src/vtx/llm/dynamic_models.py", |
| "top5_files": [ |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/context_governance.py", |
| "src/vtx/ui/widgets.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/llm/model_fetcher.py" |
| ], |
| "top10_files": [ |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/context_governance.py", |
| "src/vtx/ui/widgets.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/llm/model_fetcher.py", |
| "src/vtx/core/compaction.py", |
| "src/vtx/llm/context_length.py", |
| "src/vtx/context/skills.py", |
| "Site/src/content/blog/memory-compaction.ts", |
| "src/vtx/context/git.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0189 |
| }, |
| { |
| "query": "context item priority ranking", |
| "description": "Context priority", |
| "expected_file": "context", |
| "latency_ms": 7.03, |
| "top1_file": "src/vtx/ui/commands/settings.py", |
| "top5_files": [ |
| "src/vtx/ui/commands/settings.py", |
| "src/vtx/ui/input.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/ui/tree.py", |
| "src/vtx/ui/completion_ui.py" |
| ], |
| "top10_files": [ |
| "src/vtx/ui/commands/settings.py", |
| "src/vtx/ui/input.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/ui/tree.py", |
| "src/vtx/ui/completion_ui.py", |
| "src/vtx/llm/dynamic_models.py", |
| "src/vtx/sdk/items.py", |
| "src/vtx/llm/oauth/dynamic.py", |
| "src/vtx/context/skills.py", |
| "src/vtx/ui/floating_list.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": true, |
| "top1_score": 0.0186 |
| }, |
| { |
| "query": "tool dispatcher routing function calls", |
| "description": "Tool dispatcher", |
| "expected_file": "dispatcher", |
| "latency_ms": 6.54, |
| "top1_file": "src/vtx/dispatcher.py", |
| "top5_files": [ |
| "src/vtx/dispatcher.py", |
| "src/vtx/runtime.py", |
| "scripts/supercode_proxy.py", |
| "src/vtx/llm/sdk/supercode.py", |
| "src/vtx/llm/providers/supercode.py" |
| ], |
| "top10_files": [ |
| "src/vtx/dispatcher.py", |
| "src/vtx/runtime.py", |
| "scripts/supercode_proxy.py", |
| "src/vtx/llm/sdk/supercode.py", |
| "src/vtx/llm/providers/supercode.py", |
| "src/vtx/llm/tool_parser.py", |
| "src/vtx/turn.py", |
| "src/vtx/llm/base.py", |
| "src/vtx/context_governance.py", |
| "src/vtx/ui/export.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0222 |
| }, |
| { |
| "query": "dispatch tool call with arguments", |
| "description": "Tool call dispatch", |
| "expected_file": "dispatcher", |
| "latency_ms": 7.46, |
| "top1_file": "src/vtx/turn.py", |
| "top5_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/ui/tree.py", |
| "src/vtx/ui/session_ui.py", |
| "src/vtx/dispatcher.py", |
| "src/vtx/llm/base.py" |
| ], |
| "top10_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/ui/tree.py", |
| "src/vtx/ui/session_ui.py", |
| "src/vtx/dispatcher.py", |
| "src/vtx/llm/base.py", |
| "src/vtx/llm/tool_parser.py", |
| "src/vtx/tools/base.py", |
| "src/vtx/llm/sdk/supercode.py", |
| "src/vtx/tools/task.py", |
| "src/vtx/llm/providers/mock.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0184 |
| }, |
| { |
| "query": "event system publish subscribe hooks", |
| "description": "Event system", |
| "expected_file": "events", |
| "latency_ms": 6.33, |
| "top1_file": "src/vtx/hooks/loader.py", |
| "top5_files": [ |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/bridge.py", |
| "src/vtx/hooks/registry.py", |
| "src/vtx/hooks/runtime.py", |
| "src/vtx/hooks/agent_hook.py" |
| ], |
| "top10_files": [ |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/bridge.py", |
| "src/vtx/hooks/registry.py", |
| "src/vtx/hooks/runtime.py", |
| "src/vtx/hooks/agent_hook.py", |
| "src/vtx/agent_runner.py", |
| "src/vtx/turn.py", |
| "src/vtx/loop.py", |
| "Site/src/content/blog/mcp-extensions.ts", |
| "src/vtx/extensions.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0213 |
| }, |
| { |
| "query": "streaming event output to terminal", |
| "description": "Event streaming", |
| "expected_file": "events", |
| "latency_ms": 6.67, |
| "top1_file": "src/vtx/turn.py", |
| "top5_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/ui/blocks.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "Site/src/components/Hero.tsx" |
| ], |
| "top10_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/ui/blocks.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "Site/src/components/Hero.tsx", |
| "src/vtx/tools/task.py", |
| "src/vtx/events.py", |
| "Site/src/components/TerminalBlock.tsx", |
| "src/vtx/extensions.py", |
| "benchmark.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": true, |
| "top1_score": 0.0185 |
| }, |
| { |
| "query": "git branch creation and switching", |
| "description": "Git branch", |
| "expected_file": "git_branch", |
| "latency_ms": 6.88, |
| "top1_file": "src/vtx/git_branch.py", |
| "top5_files": [ |
| "src/vtx/git_branch.py", |
| "src/vtx/context/git.py", |
| "src/vtx/ui/widgets.py", |
| "src/vtx/ui/startup.py", |
| "scripts/install.sh" |
| ], |
| "top10_files": [ |
| "src/vtx/git_branch.py", |
| "src/vtx/context/git.py", |
| "src/vtx/ui/widgets.py", |
| "src/vtx/ui/startup.py", |
| "scripts/install.sh", |
| "scripts/install.ps1", |
| "src/vtx/prompts/builder.py", |
| "src/vtx/ui/commands/settings.py", |
| "src/vtx/git_branch.py", |
| "src/vtx/prompts/identity.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0286 |
| }, |
| { |
| "query": "github CLI integration shell command", |
| "description": "GitHub CLI", |
| "expected_file": "gh_cli", |
| "latency_ms": 6.06, |
| "top1_file": "src/vtx/tools/bash.py", |
| "top5_files": [ |
| "src/vtx/tools/bash.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/agent_runner.py", |
| "scripts/install.sh", |
| "scripts/install.ps1" |
| ], |
| "top10_files": [ |
| "src/vtx/tools/bash.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/agent_runner.py", |
| "scripts/install.sh", |
| "scripts/install.ps1", |
| "src/vtx/ui/input.py", |
| "src/vtx/llm/oauth/copilot.py", |
| "src/vtx/ui/formatting.py", |
| "src/vtx/context/git.py", |
| "src/vtx/ui/commands/agents.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0165 |
| }, |
| { |
| "query": "diff display unified patch format", |
| "description": "Diff display", |
| "expected_file": "diff_display", |
| "latency_ms": 6.21, |
| "top1_file": "src/vtx/tools/edit.py", |
| "top5_files": [ |
| "src/vtx/tools/edit.py", |
| "src/vtx/ui/blocks.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/diff_display.py", |
| "src/vtx/tools/bash.py" |
| ], |
| "top10_files": [ |
| "src/vtx/tools/edit.py", |
| "src/vtx/ui/blocks.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/diff_display.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/core/types.py", |
| "src/vtx/tools/skill.py", |
| "benchmark.py", |
| "src/vtx/ui/widgets.py", |
| "src/vtx/themes.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0191 |
| }, |
| { |
| "query": "show file changes colored diff output", |
| "description": "Colored diff", |
| "expected_file": "diff_display", |
| "latency_ms": 5.67, |
| "top1_file": "src/vtx/ui/widgets.py", |
| "top5_files": [ |
| "src/vtx/ui/widgets.py", |
| "src/vtx/tools/edit.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/ui/launch.py", |
| "src/vtx/ui/blocks.py" |
| ], |
| "top10_files": [ |
| "src/vtx/ui/widgets.py", |
| "src/vtx/tools/edit.py", |
| "src/vtx/tools/write.py", |
| "src/vtx/ui/launch.py", |
| "src/vtx/ui/blocks.py", |
| "src/vtx/ui/commands/settings.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/ui/selection_mode.py", |
| "src/vtx/config.py", |
| "src/vtx/ui/chat.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0192 |
| }, |
| { |
| "query": "plugin extension registration loading", |
| "description": "Extension loading", |
| "expected_file": "extensions", |
| "latency_ms": 6.21, |
| "top1_file": "src/vtx/extensions.py", |
| "top5_files": [ |
| "src/vtx/extensions.py", |
| "Site/src/content/blog/mcp-extensions.ts", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/hooks/registry.py", |
| "src/vtx/ui/app.py" |
| ], |
| "top10_files": [ |
| "src/vtx/extensions.py", |
| "Site/src/content/blog/mcp-extensions.ts", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/hooks/registry.py", |
| "src/vtx/ui/app.py", |
| "src/vtx/hooks/loader.py", |
| "src/vtx/extensions.py", |
| "benchmark.py", |
| "Site/scripts/prerender.mjs", |
| "src/vtx/hooks/types.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0231 |
| }, |
| { |
| "query": "builtin skill definition and registration", |
| "description": "Builtin skills", |
| "expected_file": "builtin_skill", |
| "latency_ms": 7.28, |
| "top1_file": "src/vtx/context/skills.py", |
| "top5_files": [ |
| "src/vtx/context/skills.py", |
| "src/vtx/tools/skill.py", |
| "src/vtx/sdk/skills.py", |
| "src/vtx/context/loader.py", |
| "Site/src/content/blog/agentskill.ts" |
| ], |
| "top10_files": [ |
| "src/vtx/context/skills.py", |
| "src/vtx/tools/skill.py", |
| "src/vtx/sdk/skills.py", |
| "src/vtx/context/loader.py", |
| "Site/src/content/blog/agentskill.ts", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/app.py", |
| "src/vtx/agents/registry.py", |
| "src/vtx/context/skills.py", |
| "src/vtx/tools/skill.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0226 |
| }, |
| { |
| "query": "skill slash command frontmatter parsing", |
| "description": "Skill frontmatter", |
| "expected_file": "builtin_skill", |
| "latency_ms": 5.74, |
| "top1_file": "src/vtx/context/skills.py", |
| "top5_files": [ |
| "src/vtx/context/skills.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/app.py", |
| "Site/src/content/blog/agentskill.ts", |
| "src/vtx/ui/autocomplete.py" |
| ], |
| "top10_files": [ |
| "src/vtx/context/skills.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/ui/app.py", |
| "Site/src/content/blog/agentskill.ts", |
| "src/vtx/ui/autocomplete.py", |
| "src/vtx/tools/skill.py", |
| "src/vtx/ui/input.py", |
| "src/vtx/sdk/skills.py", |
| "src/vtx/ui/commands/agents.py", |
| "src/vtx/context/skills.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0212 |
| }, |
| { |
| "query": "subagent invocation and communication", |
| "description": "Subagent invocation", |
| "expected_file": "agents", |
| "latency_ms": 6.02, |
| "top1_file": "src/vtx/tools/task.py", |
| "top5_files": [ |
| "src/vtx/tools/task.py", |
| "src/vtx/config.py", |
| "src/vtx/dispatcher.py", |
| "benchmark.py", |
| "src/vtx/tools/background.py" |
| ], |
| "top10_files": [ |
| "src/vtx/tools/task.py", |
| "src/vtx/config.py", |
| "src/vtx/dispatcher.py", |
| "benchmark.py", |
| "src/vtx/tools/background.py", |
| "src/vtx/extensions.py", |
| "src/vtx/tools/ask_user.py", |
| "src/vtx/ui/chat.py", |
| "src/vtx/loop.py", |
| "src/vtx/llm/sdk/supercode.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.017 |
| }, |
| { |
| "query": "agent message passing protocol", |
| "description": "Agent messaging", |
| "expected_file": "agents", |
| "latency_ms": 7.92, |
| "top1_file": "src/vtx/ui/commands/agents.py", |
| "top5_files": [ |
| "src/vtx/ui/commands/agents.py", |
| "src/vtx/agents/loader.py", |
| "src/vtx/ui/app_protocol.py", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/agents/api.py" |
| ], |
| "top10_files": [ |
| "src/vtx/ui/commands/agents.py", |
| "src/vtx/agents/loader.py", |
| "src/vtx/ui/app_protocol.py", |
| "src/vtx/sdk/runner.py", |
| "src/vtx/agents/api.py", |
| "src/vtx/agent_runner.py", |
| "src/vtx/runtime.py", |
| "src/vtx/hooks/agent_hook.py", |
| "src/vtx/agents/registry.py", |
| "src/vtx/ui/agent_runner.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0198 |
| }, |
| { |
| "query": "pre-commit hook integration", |
| "description": "Pre-commit hooks", |
| "expected_file": "hooks", |
| "latency_ms": 7.1, |
| "top1_file": "src/vtx/hooks/bridge.py", |
| "top5_files": [ |
| "src/vtx/hooks/bridge.py", |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/types.py", |
| "src/vtx/hooks/agent_hook.py", |
| "src/vtx/hooks/registry.py" |
| ], |
| "top10_files": [ |
| "src/vtx/hooks/bridge.py", |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/types.py", |
| "src/vtx/hooks/agent_hook.py", |
| "src/vtx/hooks/registry.py", |
| "Site/scripts/prerender.mjs", |
| "src/vtx/hooks/runtime.py", |
| "src/vtx/hooks/__init__.py", |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/bridge.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0224 |
| }, |
| { |
| "query": "async utility helpers gather timeout", |
| "description": "Async utilities", |
| "expected_file": "async_utils", |
| "latency_ms": 11.93, |
| "top1_file": "src/vtx/turn.py", |
| "top5_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/hooks/bridge.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/llm/sdk/anthropic.py", |
| "src/vtx/tools/background.py" |
| ], |
| "top10_files": [ |
| "src/vtx/turn.py", |
| "src/vtx/hooks/bridge.py", |
| "src/vtx/tools/bash.py", |
| "src/vtx/llm/sdk/anthropic.py", |
| "src/vtx/tools/background.py", |
| "src/vtx/tools/web.py", |
| "src/vtx/tools/_tool_utils.py", |
| "src/vtx/llm/oauth/copilot.py", |
| "src/vtx/llm/providers/mock.py", |
| "benchmark.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": false, |
| "hit_at_10": false, |
| "top1_score": 0.0181 |
| }, |
| { |
| "query": "headless non-interactive execution mode", |
| "description": "Headless mode", |
| "expected_file": "headless", |
| "latency_ms": 15.42, |
| "top1_file": "src/vtx/headless.py", |
| "top5_files": [ |
| "src/vtx/headless.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "Site/src/hooks/useRouteMeta.ts", |
| "src/vtx/prompts/identity.py", |
| "Site/src/components/Capabilities.tsx" |
| ], |
| "top10_files": [ |
| "src/vtx/headless.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "Site/src/hooks/useRouteMeta.ts", |
| "src/vtx/prompts/identity.py", |
| "Site/src/components/Capabilities.tsx", |
| "src/vtx/cli.py", |
| "src/vtx/ui/commands/settings.py", |
| "src/vtx/ui/app.py", |
| "benchmark.py", |
| "src/vtx/runtime.py" |
| ], |
| "hit_at_1": true, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0198 |
| }, |
| { |
| "query": "error handling exception retry logic", |
| "description": "Generic: error handling (open)", |
| "expected_file": null, |
| "latency_ms": 14.02, |
| "top1_file": "src/vtx/llm/rate_limit.py", |
| "top5_files": [ |
| "src/vtx/llm/rate_limit.py", |
| "src/vtx/core/errors.py", |
| "src/vtx/turn.py", |
| "src/vtx/llm/sdk/openai.py", |
| "src/vtx/llm/sdk/anthropic.py" |
| ], |
| "top10_files": [ |
| "src/vtx/llm/rate_limit.py", |
| "src/vtx/core/errors.py", |
| "src/vtx/turn.py", |
| "src/vtx/llm/sdk/openai.py", |
| "src/vtx/llm/sdk/anthropic.py", |
| "src/vtx/llm/providers/openai_sdk.py", |
| "src/vtx/llm/providers/anthropic_sdk.py", |
| "src/vtx/events.py", |
| "src/vtx/llm/base.py", |
| "src/vtx/tools/background.py" |
| ], |
| "hit_at_1": null, |
| "hit_at_5": null, |
| "hit_at_10": null, |
| "top1_score": 0.0177 |
| }, |
| { |
| "query": "logging setup structured log output", |
| "description": "Generic: logging (open)", |
| "expected_file": null, |
| "latency_ms": 10.64, |
| "top1_file": "src/vtx/loop.py", |
| "top5_files": [ |
| "src/vtx/loop.py", |
| "Site/src/content/blog/memory-compaction.ts", |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/registry.py", |
| "Site/src/components/FeaturesPage.tsx" |
| ], |
| "top10_files": [ |
| "src/vtx/loop.py", |
| "Site/src/content/blog/memory-compaction.ts", |
| "src/vtx/hooks/loader.py", |
| "src/vtx/hooks/registry.py", |
| "Site/src/components/FeaturesPage.tsx", |
| "src/vtx/llm/sdk/openai.py", |
| ".agents/skills/vtx-tmux-test/setup-test-project.sh", |
| "src/vtx/ui/session_ui.py", |
| "src/vtx/ui/app.py", |
| "scripts/install.sh" |
| ], |
| "hit_at_1": null, |
| "hit_at_5": null, |
| "hit_at_10": null, |
| "top1_score": 0.0179 |
| }, |
| { |
| "query": "test fixtures mock patching pytest", |
| "description": "Test utilities", |
| "expected_file": "tests", |
| "latency_ms": 6.85, |
| "top1_file": "benchmark.py", |
| "top5_files": [ |
| "benchmark.py", |
| "tests/llm/test_mock_provider.py", |
| "tests/ui/test_shell_command_detection.py", |
| "tests/test_rate_limit.py", |
| "tests/tools/test_web_expand_collapse.py" |
| ], |
| "top10_files": [ |
| "benchmark.py", |
| "tests/llm/test_mock_provider.py", |
| "tests/ui/test_shell_command_detection.py", |
| "tests/test_rate_limit.py", |
| "tests/tools/test_web_expand_collapse.py", |
| "tests/tools/test_subprocess_cancellation.py", |
| "tests/sdk/test_provider_field.py", |
| "tests/sdk/conftest.py", |
| "tests/tools/test_read.py", |
| "tests/test_grep_tool.py" |
| ], |
| "hit_at_1": false, |
| "hit_at_5": true, |
| "hit_at_10": true, |
| "top1_score": 0.0085 |
| } |
| ], |
| "recall_at_1": 0.4643, |
| "recall_at_5": 0.6429, |
| "recall_at_10": 0.7143, |
| "mean_latency_ms": 7.28, |
| "p50_latency_ms": 6.34, |
| "p95_latency_ms": 11.93, |
| "p99_latency_ms": 14.02, |
| "evaluated_queries": 28, |
| "open_queries": 2 |
| } |
| ] |
