Spaces:
Running
Running
File size: 4,205 Bytes
e0f5bb6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | """Route 2: reconstruct and audit the paper-reported Inception Score tables."""
from __future__ import annotations
import json
from pathlib import Path
import numpy as np
SETTINGS = ("ResNet CIFAR-10", "ResNet STL-10", "CNN CIFAR-10", "CNN STL-10")
BETA = np.array([-0.5, -0.3, -0.2, 0.0, 0.2, 0.3, 0.5])
BETA_IS = {
"ResNet CIFAR-10": [7.002, 7.087, 7.020, 7.022, 5.217, 4.698, 4.160],
"ResNet STL-10": [6.878, 7.181, 6.969, 6.445, 5.565, 4.858, 4.447],
"CNN CIFAR-10": [6.761, 7.010, 7.062, 6.804, 6.519, 6.322, 4.942],
"CNN STL-10": [7.520, 7.791, 7.383, 7.594, 7.302, 7.178, 6.775],
}
RHO = np.array([0.3, 0.5, 0.7, 0.9])
RHO_IS = {
"ResNet CIFAR-10": [6.265, 6.308, 6.483, 7.087],
"ResNet STL-10": [5.571, 6.335, 6.486, 7.187],
"CNN CIFAR-10": [6.280, 6.685, 6.809, 7.010],
"CNN STL-10": [6.541, 6.775, 7.332, 7.791],
}
def ranks(values: np.ndarray) -> np.ndarray:
return np.argsort(np.argsort(values)).astype(float)
def run(output_dir: Path) -> dict[str, object]:
output_dir.mkdir(parents=True, exist_ok=True)
beta_summaries: list[dict[str, object]] = []
rho_summaries: list[dict[str, object]] = []
for setting in SETTINGS:
beta_values = np.asarray(BETA_IS[setting])
beta_summaries.append(
{
"setting": setting,
"spearman_beta_vs_is": float(
np.corrcoef(ranks(BETA), ranks(beta_values))[0, 1]
),
"strict_is_decrease_as_beta_increases": bool(
np.all(np.diff(beta_values) < 0.0)
),
"best_beta": float(BETA[np.argmax(beta_values)]),
}
)
rho_values = np.asarray(RHO_IS[setting])
rho_summaries.append(
{
"setting": setting,
"strict_is_increase_as_rho_increases": bool(
np.all(np.diff(rho_values) > 0.0)
),
"spearman_rho_vs_is": float(
np.corrcoef(ranks(RHO), ranks(rho_values))[0, 1]
),
}
)
beta_zero_index = int(np.where(BETA == 0.0)[0][0])
beta_minus_point_three_index = int(np.where(BETA == -0.3)[0][0])
fixed_beta_consistency = []
for setting in SETTINGS:
table2_rho_point9 = RHO_IS[setting][-1]
fixed_beta_consistency.append(
{
"setting": setting,
"table2_rho_point9": table2_rho_point9,
"table1_beta_zero": BETA_IS[setting][beta_zero_index],
"table1_beta_minus_point_three": BETA_IS[setting][
beta_minus_point_three_index
],
"distance_to_beta_zero": abs(
table2_rho_point9 - BETA_IS[setting][beta_zero_index]
),
"distance_to_beta_minus_point_three": abs(
table2_rho_point9
- BETA_IS[setting][beta_minus_point_three_index]
),
}
)
payload = {
"claim": 5,
"route": 2,
"route_name": "reported-table consistency and association audit",
"source_hashes": {
"Table_1.tex": "fa81759aa1061d92d58eae6c060b53595648567dbf2273857fbd589ca9dc361e",
"Table_2.tex": "90fae30aa8711bd0033d3af97d9b3ec480536dae2aba4540851cbeea80f899bd",
},
"beta_values_ascending": BETA.tolist(),
"beta_inception_scores": BETA_IS,
"rho_values_ascending": RHO.tolist(),
"rho_inception_scores": RHO_IS,
"beta_summaries": beta_summaries,
"rho_summaries": rho_summaries,
"fixed_beta_consistency": fixed_beta_consistency,
"paper_text_fixed_beta_for_rho_sweep": 0.0,
"verdict": "BLOCKED",
"reason": (
"Reported aggregate IS values support directional association, but "
"are not reproduction data and Table 2's rho=0.9 values align with "
"the beta=-0.3 row rather than the text's fixed beta=0 setting."
),
}
(output_dir / "claim5_route2_table_audit.json").write_text(
json.dumps(payload, indent=2) + "\n"
)
return payload
|