| """Smoke #1: ProcessBench metric equals hand-computed values.""" | |
| import math | |
| from mathcompose.eval.processbench import score_subset, harmonic_mean | |
| def test_harmonic_mean(): | |
| assert harmonic_mean(0.5, 0.5) == 0.5 | |
| assert harmonic_mean(0.0, 0.9) == 0.0 | |
| assert math.isclose(harmonic_mean(0.6, 0.9), 2 * 0.6 * 0.9 / 1.5) | |
| def test_score_subset_handcomputed(): | |
| # correct err@2 err@0 correct | |
| golds = [-1, 2, 0, -1] | |
| preds = [-1, 2, 1, 3] | |
| s = score_subset("demo", golds, preds) | |
| assert s.n_error == 2 and s.n_correct == 2 | |
| assert math.isclose(s.acc_error, 0.5) # only err@2 hit | |
| assert math.isclose(s.acc_correct, 0.5) # only first -1 hit | |
| assert math.isclose(s.f1, 0.5) | |
| def test_all_correct_subset(): | |
| s = score_subset("c", [-1, -1], [-1, -1]) | |
| assert s.acc_correct == 1.0 | |
| assert s.acc_error == 0.0 # no erroneous samples | |
| assert s.f1 == 0.0 # harmonic mean with a 0 term | |