Commit
·
0b7bbfd
1
Parent(s):
f734731
Rename to SWE-bench Costs Calculator, open all accordions by default (v0.3.44)
Browse files- Fix ONE TRAJECTORY graphs appearing on first Load & Analyze
- Open all accordion sections by default
- Rename app title to "SWE-bench Costs Calculator"
app.py
CHANGED
|
@@ -1839,7 +1839,7 @@ def build_app():
|
|
| 1839 |
""")
|
| 1840 |
trajectories_state = gr.State(None)
|
| 1841 |
|
| 1842 |
-
gr.Markdown("# 🧮 SWE-bench
|
| 1843 |
gr.Markdown("### *Calculate cost savings with different routing strategies.*")
|
| 1844 |
gr.Markdown("## 🎯 Select a base model for cost analysis (click a row)")
|
| 1845 |
|
|
@@ -1861,12 +1861,12 @@ def build_app():
|
|
| 1861 |
plot_steps = gr.Plot(label="Distribution of API Calls (Steps) per Trajectory")
|
| 1862 |
plot_cost = gr.Plot(label="Distribution of Cost Reported by Leaderboard ($)")
|
| 1863 |
|
| 1864 |
-
with gr.Accordion("Token counts REPORTED in the metadata of .traj files [AGGREGATED ALL]", open=
|
| 1865 |
with gr.Row():
|
| 1866 |
plot_tokens_meta = gr.Plot(label="Total Tokens by Type")
|
| 1867 |
plot_tokens_cost_meta = gr.Plot(label="Total Cost by Token Type ($)")
|
| 1868 |
|
| 1869 |
-
with gr.Accordion("Token counts REPORTED in the metadata of .traj files [AGGREGATED BY TRAJECTORY]", open=
|
| 1870 |
with gr.Row():
|
| 1871 |
plot_stacked_meta = gr.Plot(label="Tokens per Trajectory (stacked)")
|
| 1872 |
with gr.Row():
|
|
@@ -1880,18 +1880,18 @@ def build_app():
|
|
| 1880 |
with gr.Row():
|
| 1881 |
single_traj_meta_cost_plot = gr.Plot(label="Cost per Step (stacked) ($)")
|
| 1882 |
|
| 1883 |
-
with gr.Accordion("Token counts CALCULATED from .traj files [AGGREGATED ALL]", open=
|
| 1884 |
with gr.Row():
|
| 1885 |
plot_tokens_calc = gr.Plot(label="Total Tokens by Type")
|
| 1886 |
plot_tokens_cost_calc = gr.Plot(label="Total Cost by Token Type ($)")
|
| 1887 |
|
| 1888 |
-
with gr.Accordion("Token counts CALCULATED from .traj files [AGGREGATED BY TRAJECTORY]", open=
|
| 1889 |
with gr.Row():
|
| 1890 |
plot_stacked_calc = gr.Plot(label="Tokens per Trajectory (stacked)")
|
| 1891 |
with gr.Row():
|
| 1892 |
plot_cost_breakdown_calc = gr.Plot(label="Cost per Trajectory")
|
| 1893 |
|
| 1894 |
-
with gr.Accordion("Token counts CALCULATED from .traj files [ONE TRAJECTORY]", open=
|
| 1895 |
with gr.Row():
|
| 1896 |
single_traj_dropdown = gr.Dropdown(label="Select Issue", choices=[], interactive=True)
|
| 1897 |
with gr.Row():
|
|
@@ -2889,6 +2889,13 @@ def build_app():
|
|
| 2889 |
first_meta_issue = meta_issue_ids[0] if meta_issue_ids else None
|
| 2890 |
has_meta_steps = len(meta_issue_ids) > 0
|
| 2891 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2892 |
fig_single_traj_meta = None
|
| 2893 |
fig_single_traj_meta_cost = None
|
| 2894 |
if first_meta_issue and first_meta_issue in metadata_steps:
|
|
@@ -2907,8 +2914,8 @@ def build_app():
|
|
| 2907 |
gr.update(visible=True),
|
| 2908 |
gr.update(visible=True),
|
| 2909 |
gr.update(choices=issue_ids, value=first_issue),
|
| 2910 |
-
|
| 2911 |
-
|
| 2912 |
gr.update(visible=has_meta_steps),
|
| 2913 |
gr.update(choices=meta_issue_ids, value=first_meta_issue),
|
| 2914 |
fig_single_traj_meta,
|
|
@@ -2957,14 +2964,6 @@ def build_app():
|
|
| 2957 |
single_traj_meta_plot,
|
| 2958 |
single_traj_meta_cost_plot,
|
| 2959 |
],
|
| 2960 |
-
).then(
|
| 2961 |
-
fn=on_single_traj_select,
|
| 2962 |
-
inputs=[trajectories_state, single_traj_dropdown, price_input, price_cache_read, price_cache_creation, price_completion, thinking_overhead, use_cache],
|
| 2963 |
-
outputs=[single_traj_plot, single_traj_cost_plot],
|
| 2964 |
-
).then(
|
| 2965 |
-
fn=on_single_traj_meta_select,
|
| 2966 |
-
inputs=[trajectories_state, single_traj_meta_dropdown, price_input, price_cache_read, price_cache_creation, price_completion],
|
| 2967 |
-
outputs=[single_traj_meta_plot, single_traj_meta_cost_plot],
|
| 2968 |
)
|
| 2969 |
|
| 2970 |
def recalculate_costs(state_data, input_price, cache_read_price, cache_creation_price, completion_price, overhead, with_cache):
|
|
|
|
| 1839 |
""")
|
| 1840 |
trajectories_state = gr.State(None)
|
| 1841 |
|
| 1842 |
+
gr.Markdown("# 🧮 SWE-bench Costs Calculator `v0.3.44`")
|
| 1843 |
gr.Markdown("### *Calculate cost savings with different routing strategies.*")
|
| 1844 |
gr.Markdown("## 🎯 Select a base model for cost analysis (click a row)")
|
| 1845 |
|
|
|
|
| 1861 |
plot_steps = gr.Plot(label="Distribution of API Calls (Steps) per Trajectory")
|
| 1862 |
plot_cost = gr.Plot(label="Distribution of Cost Reported by Leaderboard ($)")
|
| 1863 |
|
| 1864 |
+
with gr.Accordion("Token counts REPORTED in the metadata of .traj files [AGGREGATED ALL]", open=True):
|
| 1865 |
with gr.Row():
|
| 1866 |
plot_tokens_meta = gr.Plot(label="Total Tokens by Type")
|
| 1867 |
plot_tokens_cost_meta = gr.Plot(label="Total Cost by Token Type ($)")
|
| 1868 |
|
| 1869 |
+
with gr.Accordion("Token counts REPORTED in the metadata of .traj files [AGGREGATED BY TRAJECTORY]", open=True):
|
| 1870 |
with gr.Row():
|
| 1871 |
plot_stacked_meta = gr.Plot(label="Tokens per Trajectory (stacked)")
|
| 1872 |
with gr.Row():
|
|
|
|
| 1880 |
with gr.Row():
|
| 1881 |
single_traj_meta_cost_plot = gr.Plot(label="Cost per Step (stacked) ($)")
|
| 1882 |
|
| 1883 |
+
with gr.Accordion("Token counts CALCULATED from .traj files [AGGREGATED ALL]", open=True):
|
| 1884 |
with gr.Row():
|
| 1885 |
plot_tokens_calc = gr.Plot(label="Total Tokens by Type")
|
| 1886 |
plot_tokens_cost_calc = gr.Plot(label="Total Cost by Token Type ($)")
|
| 1887 |
|
| 1888 |
+
with gr.Accordion("Token counts CALCULATED from .traj files [AGGREGATED BY TRAJECTORY]", open=True):
|
| 1889 |
with gr.Row():
|
| 1890 |
plot_stacked_calc = gr.Plot(label="Tokens per Trajectory (stacked)")
|
| 1891 |
with gr.Row():
|
| 1892 |
plot_cost_breakdown_calc = gr.Plot(label="Cost per Trajectory")
|
| 1893 |
|
| 1894 |
+
with gr.Accordion("Token counts CALCULATED from .traj files [ONE TRAJECTORY]", open=True, visible=False) as single_traj_accordion:
|
| 1895 |
with gr.Row():
|
| 1896 |
single_traj_dropdown = gr.Dropdown(label="Select Issue", choices=[], interactive=True)
|
| 1897 |
with gr.Row():
|
|
|
|
| 2889 |
first_meta_issue = meta_issue_ids[0] if meta_issue_ids else None
|
| 2890 |
has_meta_steps = len(meta_issue_ids) > 0
|
| 2891 |
|
| 2892 |
+
fig_single_traj = None
|
| 2893 |
+
fig_single_traj_cost = None
|
| 2894 |
+
if first_issue and first_issue in trajectory_steps:
|
| 2895 |
+
calc_steps = trajectory_steps[first_issue]
|
| 2896 |
+
fig_single_traj = create_single_trajectory_chart(calc_steps, overhead, with_cache)
|
| 2897 |
+
fig_single_traj_cost = create_single_trajectory_cost_chart(calc_steps, input_price, cache_read_price, cache_creation_price, completion_price, overhead, with_cache)
|
| 2898 |
+
|
| 2899 |
fig_single_traj_meta = None
|
| 2900 |
fig_single_traj_meta_cost = None
|
| 2901 |
if first_meta_issue and first_meta_issue in metadata_steps:
|
|
|
|
| 2914 |
gr.update(visible=True),
|
| 2915 |
gr.update(visible=True),
|
| 2916 |
gr.update(choices=issue_ids, value=first_issue),
|
| 2917 |
+
fig_single_traj,
|
| 2918 |
+
fig_single_traj_cost,
|
| 2919 |
gr.update(visible=has_meta_steps),
|
| 2920 |
gr.update(choices=meta_issue_ids, value=first_meta_issue),
|
| 2921 |
fig_single_traj_meta,
|
|
|
|
| 2964 |
single_traj_meta_plot,
|
| 2965 |
single_traj_meta_cost_plot,
|
| 2966 |
],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2967 |
)
|
| 2968 |
|
| 2969 |
def recalculate_costs(state_data, input_price, cache_read_price, cache_creation_price, completion_price, overhead, with_cache):
|