File size: 2,551 Bytes
9a67fbe |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
import os
import re
from collections import defaultdict
LOG_ROOT = "logs"
SUMMARY_FILE = "benchmark_summary.txt"
def parse_ci_section(content):
"""Extracts Test RMSE, MAE, and their 95% CI intervals from log content."""
rmse_match = re.search(
r"Test RMSE: ([\d.]+)\s+\(95 % CI: ([\d.]+)[–-]([\d.]+)\)", content
)
mae_match = re.search(
r"Test MAE\s*: ([\d.]+)\s+\(95 % CI: ([\d.]+)[–-]([\d.]+)\)", content
)
if not (rmse_match and mae_match):
return None
rmse, rmse_lo, rmse_hi = map(float, rmse_match.groups())
mae, mae_lo, mae_hi = map(float, mae_match.groups())
rmse_pm = (rmse_hi - rmse_lo) / 2
mae_pm = (mae_hi - mae_lo) / 2
return {"rmse": rmse, "rmse_pm": rmse_pm, "mae": mae, "mae_pm": mae_pm}
def extract_encoding(fname):
for enc in ["ecfp", "smiles", "selfies"]:
if enc in fname.lower():
return enc
return "unknown"
# Map benchmark name → list of rows
benchmark_tables = defaultdict(list)
for benchmark_dir in os.listdir(LOG_ROOT):
if not benchmark_dir.endswith("-bench"):
continue
benchmark_name = benchmark_dir.replace("-bench", "")
benchmark_path = os.path.join(LOG_ROOT, benchmark_dir)
for model in os.listdir(benchmark_path):
model_path = os.path.join(benchmark_path, model)
if not os.path.isdir(model_path):
continue
for log_file in os.listdir(model_path):
if not log_file.endswith(".txt"):
continue
encoding = extract_encoding(log_file)
with open(os.path.join(model_path, log_file), "r") as f:
content = f.read()
parsed = parse_ci_section(content)
if parsed is None:
continue
benchmark_tables[benchmark_name].append(
{"model": model, "encoding": encoding, **parsed}
)
# Write to file
with open(SUMMARY_FILE, "w") as f:
def log(line=""):
f.write(line + "\n")
print(line)
for benchmark, rows in sorted(benchmark_tables.items()):
log(f"\n{benchmark.upper()} Benchmark Summary:\n")
log(f"{'Model':<13} {'Encoding':<8} {'RMSE ±':<20} {'MAE ±':<20}")
log("-" * 65)
for row in sorted(rows, key=lambda r: (r["model"], r["encoding"])):
log(
f"{row['model']:<13} {row['encoding']:<8} "
f"{row['rmse']:.4f} ± {row['rmse_pm']:.4f} "
f"{row['mae']:.4f} ± {row['mae_pm']:.4f}"
)
|