anhtld commited on
Commit
c4f7fde
·
verified ·
1 Parent(s): 0ac5328

Auto-sync: 2026-06-30 03:10:57 (part 3)

Browse files
scripts/build_paper_analysis.py CHANGED
@@ -385,6 +385,14 @@ METHODS = [
385
  "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8_summary.json"
386
  ),
387
  ),
 
 
 
 
 
 
 
 
388
  MethodSpec(
389
  key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001",
390
  label="K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.01",
@@ -884,6 +892,15 @@ def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, sour
884
  ),
885
  }
886
  )
 
 
 
 
 
 
 
 
 
887
  return output
888
 
889
 
@@ -1099,11 +1116,21 @@ def _render_markdown(report: dict[str, Any]) -> str:
1099
  f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.",
1100
  ]
1101
  )
 
 
 
1102
  oracle_key = (
1103
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8"
1104
  )
1105
- oracle = methods.get(oracle_key)
 
 
1106
  if oracle and oracle.get("num_completed"):
 
 
 
 
 
1107
  lines.extend(
1108
  [
1109
  "",
@@ -1131,6 +1158,27 @@ def _render_markdown(report: dict[str, Any]) -> str:
1131
  ),
1132
  ]
1133
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1134
  lines.extend(
1135
  [
1136
  "",
 
385
  "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8_summary.json"
386
  ),
387
  ),
388
+ MethodSpec(
389
+ key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace",
390
+ label="K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 branch trace",
391
+ summary_path=(
392
+ "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
393
+ "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8trace_summary.json"
394
+ ),
395
+ ),
396
  MethodSpec(
397
  key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001",
398
  label="K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.01",
 
892
  ),
893
  }
894
  )
895
+ for key in (
896
+ "mean_candidate_oracle_best_branch_rank",
897
+ "candidate_oracle_best_branch_rank_counts",
898
+ "mean_candidate_oracle_branch_success_rates",
899
+ "mean_candidate_oracle_branch_progress",
900
+ "mean_candidate_oracle_branch_score_gains_over_selected",
901
+ ):
902
+ if key in data:
903
+ output[key] = data[key]
904
  return output
905
 
906
 
 
1116
  f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.",
1117
  ]
1118
  )
1119
+ trace_oracle_key = (
1120
+ "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace"
1121
+ )
1122
  oracle_key = (
1123
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8"
1124
  )
1125
+ oracle = methods.get(trace_oracle_key)
1126
+ if not oracle or not oracle.get("num_completed"):
1127
+ oracle = methods.get(oracle_key)
1128
  if oracle and oracle.get("num_completed"):
1129
+ branch_success = oracle.get("mean_candidate_oracle_branch_success_rates") or []
1130
+ branch_gains = (
1131
+ oracle.get("mean_candidate_oracle_branch_score_gains_over_selected")
1132
+ or []
1133
+ )
1134
  lines.extend(
1135
  [
1136
  "",
 
1158
  ),
1159
  ]
1160
  )
1161
+ if oracle.get("mean_candidate_oracle_best_branch_rank") is not None:
1162
+ lines.extend(
1163
+ [
1164
+ (
1165
+ "- Mean best branch rank in the field-ordered prefix: "
1166
+ f"{oracle['mean_candidate_oracle_best_branch_rank']:.2f}; "
1167
+ "rank histogram "
1168
+ f"{oracle.get('candidate_oracle_best_branch_rank_counts', {})}."
1169
+ ),
1170
+ (
1171
+ "- Branch success by prefix rank: "
1172
+ + ", ".join(_pct(value) for value in branch_success)
1173
+ + "."
1174
+ ),
1175
+ (
1176
+ "- Branch score gain by prefix rank: "
1177
+ + ", ".join(f"{value:+.3f}" for value in branch_gains)
1178
+ + "."
1179
+ ),
1180
+ ]
1181
+ )
1182
  lines.extend(
1183
  [
1184
  "",
scripts/build_paper_table_status.py CHANGED
@@ -735,6 +735,16 @@ SPECS = [
735
  story_role="diagnostic oracle over generated clean candidate prefix to separate proposal headroom from field ranking",
736
  pending_job="14953513/14953522",
737
  ),
 
 
 
 
 
 
 
 
 
 
738
  ResultSpec(
739
  key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001",
740
  label="K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.01",
@@ -1523,6 +1533,14 @@ def _decision_notes(rows: list[dict[str, Any]]) -> list[str]:
1523
  "Candidate-oracle prefix K=8 is diagnostic-only; compare it to the clean selected row "
1524
  "to decide whether the next paper move should be proposal generation or field ranking."
1525
  )
 
 
 
 
 
 
 
 
1526
  return notes
1527
 
1528
 
 
735
  story_role="diagnostic oracle over generated clean candidate prefix to separate proposal headroom from field ranking",
736
  pending_job="14953513/14953522",
737
  ),
738
+ ResultSpec(
739
+ key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace",
740
+ label="K4 compatible residual retrieval, unique candidate-oracle prefix K=8 branch trace",
741
+ path="h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8trace_summary.json",
742
+ clean_deployment="diagnostic",
743
+ same_state_proposals="no",
744
+ expert_proposal="no",
745
+ story_role="diagnostic branch trace for deployable selector calibration over the local tangent chart",
746
+ pending_job="14953960/14953961",
747
+ ),
748
  ResultSpec(
749
  key="retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001",
750
  label="K4 composed compatible residual retrieval, no-op bonus 0.03, singleton near-miss bonus 0.01",
 
1533
  "Candidate-oracle prefix K=8 is diagnostic-only; compare it to the clean selected row "
1534
  "to decide whether the next paper move should be proposal generation or field ranking."
1535
  )
1536
+ oracle_trace = by_key[
1537
+ "retrieval_residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace"
1538
+ ]
1539
+ if oracle_trace["success"] is not None:
1540
+ notes.append(
1541
+ "Candidate-oracle branch trace is diagnostic-only; use its rank histogram and branch "
1542
+ "success profile to design one clean selector-calibration run rather than a broad sweep."
1543
+ )
1544
  return notes
1545
 
1546