NathanRoll commited on
Commit
d473813
·
verified ·
1 Parent(s): 23ffae0

Use exact reference values for trivial records

Browse files
Files changed (2) hide show
  1. app.py +14 -6
  2. packing_benchmark/store.py +14 -0
app.py CHANGED
@@ -13,7 +13,7 @@ from typing import Any
13
  import gradio as gr
14
 
15
  from packing_benchmark.hub_sync import maybe_hydrate_from_dataset
16
- from packing_benchmark.store import SolutionStore, metric_matches_reference
17
  from packing_benchmark.verifier import (
18
  DEFAULT_TOLERANCE,
19
  load_solution_json,
@@ -2007,12 +2007,20 @@ def record_detail_rows(record: dict[str, Any], visual_record: dict[str, Any], so
2007
  rows: list[str] = []
2008
  rows.append(detail_row("Rendering", esc(visual_provenance(record, visual_record))))
2009
  if record.get("record_type") == "reference" and visual_record.get("verified"):
2010
- rows.append(
2011
- detail_row(
2012
- "Coordinate JSON",
2013
- f"{esc(metric_symbol(visual_record))} = {esc(metric_value(visual_record))} from the stored evaluator",
 
 
 
 
 
 
 
 
 
2014
  )
2015
- )
2016
  rows.append(
2017
  detail_row(
2018
  "Friedman reference",
 
13
  import gradio as gr
14
 
15
  from packing_benchmark.hub_sync import maybe_hydrate_from_dataset
16
+ from packing_benchmark.store import SolutionStore, is_trivial_record, metric_matches_reference
17
  from packing_benchmark.verifier import (
18
  DEFAULT_TOLERANCE,
19
  load_solution_json,
 
2007
  rows: list[str] = []
2008
  rows.append(detail_row("Rendering", esc(visual_provenance(record, visual_record))))
2009
  if record.get("record_type") == "reference" and visual_record.get("verified"):
2010
+ if is_trivial_record(record):
2011
+ rows.append(
2012
+ detail_row(
2013
+ "Coordinate JSON",
2014
+ "used only for rendering; the exact trivial value is shown as the record",
2015
+ )
2016
+ )
2017
+ else:
2018
+ rows.append(
2019
+ detail_row(
2020
+ "Coordinate JSON",
2021
+ f"{esc(metric_symbol(visual_record))} = {esc(metric_value(visual_record))} from the stored evaluator",
2022
+ )
2023
  )
 
2024
  rows.append(
2025
  detail_row(
2026
  "Friedman reference",
packing_benchmark/store.py CHANGED
@@ -50,6 +50,17 @@ def metric_matches_reference(coordinate: dict[str, Any], reference: dict[str, An
50
  return metric_float(coordinate) <= metric_float(reference) + reference_metric_tolerance(reference)
51
 
52
 
 
 
 
 
 
 
 
 
 
 
 
53
  class SolutionStore:
54
  def __init__(self, root: Path):
55
  self.root = root
@@ -147,6 +158,9 @@ class SolutionStore:
147
  if reference is None:
148
  best[key] = coordinate
149
  continue
 
 
 
150
  if metric_matches_reference(coordinate, reference):
151
  best[key] = coordinate
152
  else:
 
50
  return metric_float(coordinate) <= metric_float(reference) + reference_metric_tolerance(reference)
51
 
52
 
53
+ def is_trivial_record(record: dict[str, Any]) -> bool:
54
+ try:
55
+ if int(record.get("n") or 0) == 1:
56
+ return True
57
+ except (TypeError, ValueError):
58
+ pass
59
+ credit = str(record.get("submitted_by") or record.get("credit") or record.get("friedman_credit") or "").strip().lower()
60
+ text = str(record.get("reference_text") or record.get("friedman_reference_text") or "").lower()
61
+ return credit == "trivial" or "trivial" in text
62
+
63
+
64
  class SolutionStore:
65
  def __init__(self, root: Path):
66
  self.root = root
 
158
  if reference is None:
159
  best[key] = coordinate
160
  continue
161
+ if is_trivial_record(reference):
162
+ best[key] = reference
163
+ continue
164
  if metric_matches_reference(coordinate, reference):
165
  best[key] = coordinate
166
  else: