| { |
| "seeds": [ |
| 1004, |
| 1005, |
| 1006, |
| 1007, |
| 1008 |
| ], |
| "split_hash": "2e38193a038079d66cfb5d17dc149af12a9f63a773a2e48ee21d04bb006250fc", |
| "data_sha256": "122dfd62dcf8d46ece390ed23335e69cac0d3db8e91479c162d5e4186acdbe0d", |
| "n_eval_tasks": 88, |
| "n_preference_decisions": 968, |
| "verifier_rmse": { |
| "Egc": { |
| "mean": 0.682365035610178, |
| "sd": 0.023200213429402764, |
| "ci95": 0.028802251598823818 |
| }, |
| "Egb": { |
| "mean": 0.7795461366589093, |
| "sd": 0.03702590414781171, |
| "ci95": 0.04596636191232919 |
| } |
| }, |
| "summary": { |
| "sft": { |
| "hypervolume": { |
| "mean": 0.3263172158958249, |
| "sd": 0.007077638860046684, |
| "ci95": 0.008786640510570721 |
| }, |
| "mip": { |
| "mean": 0.5542560282390163, |
| "sd": 0.00958850553545619, |
| "ci95": 0.01190379345988773 |
| }, |
| "preference_regret": { |
| "mean": 0.03366121577648368, |
| "sd": 0.00958850553545623, |
| "ci95": 0.01190379345988778 |
| }, |
| "near_optimal": { |
| "mean": 0.7822314049586777, |
| "sd": 0.07295465955600312, |
| "ci95": 0.09057065212923908 |
| }, |
| "unique": { |
| "mean": 2.090909090909091, |
| "sd": 0.1421590454908727, |
| "ci95": 0.17648547103827913 |
| } |
| }, |
| "random": { |
| "hypervolume": { |
| "mean": 0.32798411805224303, |
| "sd": 0.004243042593788916, |
| "ci95": 0.005267588623816396 |
| }, |
| "mip": { |
| "mean": 0.5, |
| "sd": 0.0, |
| "ci95": 0.0 |
| }, |
| "preference_regret": { |
| "mean": 0.08791724401549997, |
| "sd": 6.206335383118182e-18, |
| "ci95": 7.704947790898585e-18 |
| }, |
| "near_optimal": { |
| "mean": 0.4483471074380165, |
| "sd": 0.015059276587926896, |
| "ci95": 0.018695563922357987 |
| }, |
| "unique": { |
| "mean": 4.409090909090909, |
| "sd": 0.2249885212866286, |
| "ci95": 0.2793153613290541 |
| } |
| }, |
| "greedy_verifier": { |
| "hypervolume": { |
| "mean": 0.3374837855543042, |
| "sd": 0.0030145483976481056, |
| "ci95": 0.0037424561489530685 |
| }, |
| "mip": { |
| "mean": 0.5684625263089786, |
| "sd": 0.002066564745402613, |
| "ci95": 0.002565567679946882 |
| }, |
| "preference_regret": { |
| "mean": 0.019454717706521275, |
| "sd": 0.002066564745402651, |
| "ci95": 0.0025655676799469294 |
| }, |
| "near_optimal": { |
| "mean": 0.8334710743801652, |
| "sd": 0.01664794442821267, |
| "ci95": 0.020667839349138354 |
| }, |
| "unique": { |
| "mean": 2.0113636363636362, |
| "sd": 0.1041494476126327, |
| "ci95": 0.1292978878468356 |
| } |
| }, |
| "oracle_grid": { |
| "hypervolume": { |
| "mean": 0.3547658201014365, |
| "sd": 0.0, |
| "ci95": 0.0 |
| }, |
| "mip": { |
| "mean": 0.5879172440154999, |
| "sd": 0.0, |
| "ci95": 0.0 |
| }, |
| "preference_regret": { |
| "mean": 0.0, |
| "sd": 0.0, |
| "ci95": 0.0 |
| }, |
| "near_optimal": { |
| "mean": 1.0, |
| "sd": 0.0, |
| "ci95": 0.0 |
| }, |
| "unique": { |
| "mean": 1.9772727272727273, |
| "sd": 0.0, |
| "ci95": 0.0 |
| } |
| } |
| }, |
| "by_direction": { |
| "up_up": { |
| "sft": { |
| "hypervolume": { |
| "mean": 0.5547866157591552, |
| "sd": 0.004451111549643458, |
| "ci95": 0.005525898937842734 |
| }, |
| "mip": { |
| "mean": 0.7173941342744271, |
| "sd": 0.006962489498633501, |
| "ci95": 0.008643686615385294 |
| } |
| }, |
| "random": { |
| "hypervolume": { |
| "mean": 0.5300234743935974, |
| "sd": 0.008807646321860372, |
| "ci95": 0.010934384122267451 |
| }, |
| "mip": { |
| "mean": 0.62319888712802, |
| "sd": 0.00339621218061192, |
| "ci95": 0.0042162783547932525 |
| } |
| }, |
| "greedy_verifier": { |
| "hypervolume": { |
| "mean": 0.5415841777575114, |
| "sd": 0.008233289246674151, |
| "ci95": 0.010221339949746493 |
| }, |
| "mip": { |
| "mean": 0.7101234331357681, |
| "sd": 0.004102313866773968, |
| "ci95": 0.005092878843020596 |
| } |
| }, |
| "oracle_grid": { |
| "hypervolume": { |
| "mean": 0.5718122839434651, |
| "sd": 0.0, |
| "ci95": 0.0 |
| }, |
| "mip": { |
| "mean": 0.7311801380469454, |
| "sd": 0.0, |
| "ci95": 0.0 |
| } |
| }, |
| "sft_minus_greedy": { |
| "hypervolume": { |
| "mean": 0.013202438001643802, |
| "sd": 0.011560547874820054, |
| "ci95": 0.01435201488658834 |
| }, |
| "mip": { |
| "mean": 0.0072707011386590635, |
| "sd": 0.009143332516037751, |
| "ci95": 0.0113511267635526 |
| } |
| } |
| }, |
| "up_down": { |
| "sft": { |
| "hypervolume": { |
| "mean": 0.26637109162862416, |
| "sd": 0.008435619826089943, |
| "ci95": 0.010472526270605242 |
| }, |
| "mip": { |
| "mean": 0.5324022327044661, |
| "sd": 0.004339669649482129, |
| "ci95": 0.005387547725821998 |
| } |
| }, |
| "random": { |
| "hypervolume": { |
| "mean": 0.2768259355647035, |
| "sd": 0.0035200122661316934, |
| "ci95": 0.004369971820672209 |
| }, |
| "mip": { |
| "mean": 0.4991439280925797, |
| "sd": 0.0030486857043436617, |
| "ci95": 0.0037848364183994494 |
| } |
| }, |
| "greedy_verifier": { |
| "hypervolume": { |
| "mean": 0.2812522324658844, |
| "sd": 0.003391659625414806, |
| "ci95": 0.004210626517123577 |
| }, |
| "mip": { |
| "mean": 0.5487054102595932, |
| "sd": 0.0018000664676603043, |
| "ci95": 0.0022347194112641747 |
| } |
| }, |
| "oracle_grid": { |
| "hypervolume": { |
| "mean": 0.29986543339069294, |
| "sd": 0.0, |
| "ci95": 0.0 |
| }, |
| "mip": { |
| "mean": 0.5622086416925245, |
| "sd": 0.0, |
| "ci95": 0.0 |
| } |
| }, |
| "sft_minus_greedy": { |
| "hypervolume": { |
| "mean": -0.014881140837260253, |
| "sd": 0.0057456736527104965, |
| "ci95": 0.007133052402887352 |
| }, |
| "mip": { |
| "mean": -0.016303177555127068, |
| "sd": 0.0043833494770607615, |
| "ci95": 0.005441774700394509 |
| } |
| } |
| }, |
| "down_up": { |
| "sft": { |
| "hypervolume": { |
| "mean": 0.26823553613576073, |
| "sd": 0.0136372196937986, |
| "ci95": 0.01693013014403694 |
| }, |
| "mip": { |
| "mean": 0.5337835686465041, |
| "sd": 0.01378559041706564, |
| "ci95": 0.01711432719525979 |
| } |
| }, |
| "random": { |
| "hypervolume": { |
| "mean": 0.273847825696121, |
| "sd": 0.0032118959376035605, |
| "ci95": 0.003987456201021654 |
| }, |
| "mip": { |
| "mean": 0.5008560719074203, |
| "sd": 0.0030486857043436825, |
| "ci95": 0.003784836418399475 |
| } |
| }, |
| "greedy_verifier": { |
| "hypervolume": { |
| "mean": 0.28263117534455195, |
| "sd": 0.007162604591140678, |
| "ci95": 0.008892122486919516 |
| }, |
| "mip": { |
| "mean": 0.5490746269413723, |
| "sd": 0.0033286869480678356, |
| "ci95": 0.004132448145949615 |
| } |
| }, |
| "oracle_grid": { |
| "hypervolume": { |
| "mean": 0.2864046084806097, |
| "sd": 0.0, |
| "ci95": 0.0 |
| }, |
| "mip": { |
| "mean": 0.5687956961246301, |
| "sd": 0.0, |
| "ci95": 0.0 |
| } |
| }, |
| "sft_minus_greedy": { |
| "hypervolume": { |
| "mean": -0.014395639208791217, |
| "sd": 0.01697110797142767, |
| "ci95": 0.021069035558283978 |
| }, |
| "mip": { |
| "mean": -0.015291058294868076, |
| "sd": 0.014647717733847185, |
| "ci95": 0.01818462803381549 |
| } |
| } |
| }, |
| "down_down": { |
| "sft": { |
| "hypervolume": { |
| "mean": 0.21587562005975972, |
| "sd": 0.011945547963630178, |
| "ci95": 0.014829978999169532 |
| }, |
| "mip": { |
| "mean": 0.43344417733066765, |
| "sd": 0.015954219291847732, |
| "ci95": 0.019806603913576 |
| } |
| }, |
| "random": { |
| "hypervolume": { |
| "mean": 0.23123923655455006, |
| "sd": 0.00532492595981477, |
| "ci95": 0.0066107088931052806 |
| }, |
| "mip": { |
| "mean": 0.3768011128719801, |
| "sd": 0.003396212180611927, |
| "ci95": 0.004216278354793261 |
| } |
| }, |
| "greedy_verifier": { |
| "hypervolume": { |
| "mean": 0.24446755664926903, |
| "sd": 0.002140164385024888, |
| "ci95": 0.0026569390522161113 |
| }, |
| "mip": { |
| "mean": 0.46594663489918126, |
| "sd": 0.0020302006151077918, |
| "ci95": 0.0025204228870719827 |
| } |
| }, |
| "oracle_grid": { |
| "hypervolume": { |
| "mean": 0.2609809545909782, |
| "sd": 0.0, |
| "ci95": 0.0 |
| }, |
| "mip": { |
| "mean": 0.48948450019789985, |
| "sd": 0.0, |
| "ci95": 0.0 |
| } |
| }, |
| "sft_minus_greedy": { |
| "hypervolume": { |
| "mean": -0.028591936589509294, |
| "sd": 0.012851095799986063, |
| "ci95": 0.01595418489050146 |
| }, |
| "mip": { |
| "mean": -0.032502457568513596, |
| "sd": 0.016879600284616415, |
| "ci95": 0.020955431973265923 |
| } |
| } |
| } |
| }, |
| "sft_minus": { |
| "random": { |
| "hypervolume": { |
| "mean": -0.0016669021564180487, |
| "sd": 0.010411980805258838, |
| "ci95": 0.012926109137217073 |
| }, |
| "mip": { |
| "mean": 0.05425602823901628, |
| "sd": 0.00958850553545619, |
| "ci95": 0.01190379345988773 |
| }, |
| "preference_regret": { |
| "mean": -0.054256028239016284, |
| "sd": 0.00958850553545623, |
| "ci95": 0.01190379345988778 |
| }, |
| "near_optimal": { |
| "mean": 0.3338842975206612, |
| "sd": 0.07947633038235787, |
| "ci95": 0.09866707781760457 |
| }, |
| "unique": { |
| "mean": -2.3181818181818183, |
| "sd": 0.2821524776803439, |
| "ci95": 0.35028240908687147 |
| } |
| }, |
| "greedy_verifier": { |
| "hypervolume": { |
| "mean": -0.011166569658479253, |
| "sd": 0.008460070908431904, |
| "ci95": 0.010502881432104929 |
| }, |
| "mip": { |
| "mean": -0.014206498069962437, |
| "sd": 0.010572809308625373, |
| "ci95": 0.013125772085677476 |
| }, |
| "preference_regret": { |
| "mean": 0.014206498069962404, |
| "sd": 0.010572809308625449, |
| "ci95": 0.013125772085677572 |
| }, |
| "near_optimal": { |
| "mean": -0.05123966942148759, |
| "sd": 0.0774607415697123, |
| "ci95": 0.09616479497101585 |
| }, |
| "unique": { |
| "mean": 0.0795454545454544, |
| "sd": 0.11220691851211084, |
| "ci95": 0.1393009554825347 |
| } |
| } |
| }, |
| "sft_relative_percent": { |
| "random": { |
| "hypervolume": { |
| "mean": -0.4802110660459289, |
| "sd": 3.1398164434122986, |
| "ci95": 3.897972036010412 |
| }, |
| "mip": { |
| "mean": 10.851205647803255, |
| "sd": 1.9177011070912375, |
| "ci95": 2.3807586919775456 |
| } |
| }, |
| "greedy_verifier": { |
| "hypervolume": { |
| "mean": -3.298275440887257, |
| "sd": 2.4788612860440433, |
| "ci95": 3.0774193804932795 |
| }, |
| "mip": { |
| "mean": -2.496162558744969, |
| "sd": 1.8505323175982593, |
| "ci95": 2.2973709946853575 |
| } |
| } |
| } |
| } |
|
|
