IgorSlinko commited on
Commit
0b7bbfd
·
1 Parent(s): f734731

Rename to SWE-bench Costs Calculator, open all accordions by default (v0.3.44)

Browse files

- Fix ONE TRAJECTORY graphs appearing on first Load & Analyze
- Open all accordion sections by default
- Rename app title to "SWE-bench Costs Calculator"

Files changed (1) hide show
  1. app.py +15 -16
app.py CHANGED
@@ -1839,7 +1839,7 @@ def build_app():
1839
  """)
1840
  trajectories_state = gr.State(None)
1841
 
1842
- gr.Markdown("# 🧮 SWE-bench analytics tool `v0.3.43`")
1843
  gr.Markdown("### *Calculate cost savings with different routing strategies.*")
1844
  gr.Markdown("## 🎯 Select a base model for cost analysis (click a row)")
1845
 
@@ -1861,12 +1861,12 @@ def build_app():
1861
  plot_steps = gr.Plot(label="Distribution of API Calls (Steps) per Trajectory")
1862
  plot_cost = gr.Plot(label="Distribution of Cost Reported by Leaderboard ($)")
1863
 
1864
- with gr.Accordion("Token counts REPORTED in the metadata of .traj files [AGGREGATED ALL]", open=False):
1865
  with gr.Row():
1866
  plot_tokens_meta = gr.Plot(label="Total Tokens by Type")
1867
  plot_tokens_cost_meta = gr.Plot(label="Total Cost by Token Type ($)")
1868
 
1869
- with gr.Accordion("Token counts REPORTED in the metadata of .traj files [AGGREGATED BY TRAJECTORY]", open=False):
1870
  with gr.Row():
1871
  plot_stacked_meta = gr.Plot(label="Tokens per Trajectory (stacked)")
1872
  with gr.Row():
@@ -1880,18 +1880,18 @@ def build_app():
1880
  with gr.Row():
1881
  single_traj_meta_cost_plot = gr.Plot(label="Cost per Step (stacked) ($)")
1882
 
1883
- with gr.Accordion("Token counts CALCULATED from .traj files [AGGREGATED ALL]", open=False):
1884
  with gr.Row():
1885
  plot_tokens_calc = gr.Plot(label="Total Tokens by Type")
1886
  plot_tokens_cost_calc = gr.Plot(label="Total Cost by Token Type ($)")
1887
 
1888
- with gr.Accordion("Token counts CALCULATED from .traj files [AGGREGATED BY TRAJECTORY]", open=False):
1889
  with gr.Row():
1890
  plot_stacked_calc = gr.Plot(label="Tokens per Trajectory (stacked)")
1891
  with gr.Row():
1892
  plot_cost_breakdown_calc = gr.Plot(label="Cost per Trajectory")
1893
 
1894
- with gr.Accordion("Token counts CALCULATED from .traj files [ONE TRAJECTORY]", open=False, visible=False) as single_traj_accordion:
1895
  with gr.Row():
1896
  single_traj_dropdown = gr.Dropdown(label="Select Issue", choices=[], interactive=True)
1897
  with gr.Row():
@@ -2889,6 +2889,13 @@ def build_app():
2889
  first_meta_issue = meta_issue_ids[0] if meta_issue_ids else None
2890
  has_meta_steps = len(meta_issue_ids) > 0
2891
 
 
 
 
 
 
 
 
2892
  fig_single_traj_meta = None
2893
  fig_single_traj_meta_cost = None
2894
  if first_meta_issue and first_meta_issue in metadata_steps:
@@ -2907,8 +2914,8 @@ def build_app():
2907
  gr.update(visible=True),
2908
  gr.update(visible=True),
2909
  gr.update(choices=issue_ids, value=first_issue),
2910
- gr.update(),
2911
- gr.update(),
2912
  gr.update(visible=has_meta_steps),
2913
  gr.update(choices=meta_issue_ids, value=first_meta_issue),
2914
  fig_single_traj_meta,
@@ -2957,14 +2964,6 @@ def build_app():
2957
  single_traj_meta_plot,
2958
  single_traj_meta_cost_plot,
2959
  ],
2960
- ).then(
2961
- fn=on_single_traj_select,
2962
- inputs=[trajectories_state, single_traj_dropdown, price_input, price_cache_read, price_cache_creation, price_completion, thinking_overhead, use_cache],
2963
- outputs=[single_traj_plot, single_traj_cost_plot],
2964
- ).then(
2965
- fn=on_single_traj_meta_select,
2966
- inputs=[trajectories_state, single_traj_meta_dropdown, price_input, price_cache_read, price_cache_creation, price_completion],
2967
- outputs=[single_traj_meta_plot, single_traj_meta_cost_plot],
2968
  )
2969
 
2970
  def recalculate_costs(state_data, input_price, cache_read_price, cache_creation_price, completion_price, overhead, with_cache):
 
1839
  """)
1840
  trajectories_state = gr.State(None)
1841
 
1842
+ gr.Markdown("# 🧮 SWE-bench Costs Calculator `v0.3.44`")
1843
  gr.Markdown("### *Calculate cost savings with different routing strategies.*")
1844
  gr.Markdown("## 🎯 Select a base model for cost analysis (click a row)")
1845
 
 
1861
  plot_steps = gr.Plot(label="Distribution of API Calls (Steps) per Trajectory")
1862
  plot_cost = gr.Plot(label="Distribution of Cost Reported by Leaderboard ($)")
1863
 
1864
+ with gr.Accordion("Token counts REPORTED in the metadata of .traj files [AGGREGATED ALL]", open=True):
1865
  with gr.Row():
1866
  plot_tokens_meta = gr.Plot(label="Total Tokens by Type")
1867
  plot_tokens_cost_meta = gr.Plot(label="Total Cost by Token Type ($)")
1868
 
1869
+ with gr.Accordion("Token counts REPORTED in the metadata of .traj files [AGGREGATED BY TRAJECTORY]", open=True):
1870
  with gr.Row():
1871
  plot_stacked_meta = gr.Plot(label="Tokens per Trajectory (stacked)")
1872
  with gr.Row():
 
1880
  with gr.Row():
1881
  single_traj_meta_cost_plot = gr.Plot(label="Cost per Step (stacked) ($)")
1882
 
1883
+ with gr.Accordion("Token counts CALCULATED from .traj files [AGGREGATED ALL]", open=True):
1884
  with gr.Row():
1885
  plot_tokens_calc = gr.Plot(label="Total Tokens by Type")
1886
  plot_tokens_cost_calc = gr.Plot(label="Total Cost by Token Type ($)")
1887
 
1888
+ with gr.Accordion("Token counts CALCULATED from .traj files [AGGREGATED BY TRAJECTORY]", open=True):
1889
  with gr.Row():
1890
  plot_stacked_calc = gr.Plot(label="Tokens per Trajectory (stacked)")
1891
  with gr.Row():
1892
  plot_cost_breakdown_calc = gr.Plot(label="Cost per Trajectory")
1893
 
1894
+ with gr.Accordion("Token counts CALCULATED from .traj files [ONE TRAJECTORY]", open=True, visible=False) as single_traj_accordion:
1895
  with gr.Row():
1896
  single_traj_dropdown = gr.Dropdown(label="Select Issue", choices=[], interactive=True)
1897
  with gr.Row():
 
2889
  first_meta_issue = meta_issue_ids[0] if meta_issue_ids else None
2890
  has_meta_steps = len(meta_issue_ids) > 0
2891
 
2892
+ fig_single_traj = None
2893
+ fig_single_traj_cost = None
2894
+ if first_issue and first_issue in trajectory_steps:
2895
+ calc_steps = trajectory_steps[first_issue]
2896
+ fig_single_traj = create_single_trajectory_chart(calc_steps, overhead, with_cache)
2897
+ fig_single_traj_cost = create_single_trajectory_cost_chart(calc_steps, input_price, cache_read_price, cache_creation_price, completion_price, overhead, with_cache)
2898
+
2899
  fig_single_traj_meta = None
2900
  fig_single_traj_meta_cost = None
2901
  if first_meta_issue and first_meta_issue in metadata_steps:
 
2914
  gr.update(visible=True),
2915
  gr.update(visible=True),
2916
  gr.update(choices=issue_ids, value=first_issue),
2917
+ fig_single_traj,
2918
+ fig_single_traj_cost,
2919
  gr.update(visible=has_meta_steps),
2920
  gr.update(choices=meta_issue_ids, value=first_meta_issue),
2921
  fig_single_traj_meta,
 
2964
  single_traj_meta_plot,
2965
  single_traj_meta_cost_plot,
2966
  ],
 
 
 
 
 
 
 
 
2967
  )
2968
 
2969
  def recalculate_costs(state_data, input_price, cache_read_price, cache_creation_price, completion_price, overhead, with_cache):