{
  "seeds": [
    1004,
    1005,
    1006,
    1007,
    1008
  ],
  "split_hash": "2e38193a038079d66cfb5d17dc149af12a9f63a773a2e48ee21d04bb006250fc",
  "data_sha256": "122dfd62dcf8d46ece390ed23335e69cac0d3db8e91479c162d5e4186acdbe0d",
  "n_eval_tasks": 88,
  "n_preference_decisions": 968,
  "verifier_rmse": {
    "Egc": {
      "mean": 0.682365035610178,
      "sd": 0.023200213429402764,
      "ci95": 0.028802251598823818
    },
    "Egb": {
      "mean": 0.7795461366589093,
      "sd": 0.03702590414781171,
      "ci95": 0.04596636191232919
    }
  },
  "summary": {
    "sft": {
      "hypervolume": {
        "mean": 0.3263172158958249,
        "sd": 0.007077638860046684,
        "ci95": 0.008786640510570721
      },
      "mip": {
        "mean": 0.5542560282390163,
        "sd": 0.00958850553545619,
        "ci95": 0.01190379345988773
      },
      "preference_regret": {
        "mean": 0.03366121577648368,
        "sd": 0.00958850553545623,
        "ci95": 0.01190379345988778
      },
      "near_optimal": {
        "mean": 0.7822314049586777,
        "sd": 0.07295465955600312,
        "ci95": 0.09057065212923908
      },
      "unique": {
        "mean": 2.090909090909091,
        "sd": 0.1421590454908727,
        "ci95": 0.17648547103827913
      }
    },
    "random": {
      "hypervolume": {
        "mean": 0.32798411805224303,
        "sd": 0.004243042593788916,
        "ci95": 0.005267588623816396
      },
      "mip": {
        "mean": 0.5,
        "sd": 0.0,
        "ci95": 0.0
      },
      "preference_regret": {
        "mean": 0.08791724401549997,
        "sd": 6.206335383118182e-18,
        "ci95": 7.704947790898585e-18
      },
      "near_optimal": {
        "mean": 0.4483471074380165,
        "sd": 0.015059276587926896,
        "ci95": 0.018695563922357987
      },
      "unique": {
        "mean": 4.409090909090909,
        "sd": 0.2249885212866286,
        "ci95": 0.2793153613290541
      }
    },
    "greedy_verifier": {
      "hypervolume": {
        "mean": 0.3374837855543042,
        "sd": 0.0030145483976481056,
        "ci95": 0.0037424561489530685
      },
      "mip": {
        "mean": 0.5684625263089786,
        "sd": 0.002066564745402613,
        "ci95": 0.002565567679946882
      },
      "preference_regret": {
        "mean": 0.019454717706521275,
        "sd": 0.002066564745402651,
        "ci95": 0.0025655676799469294
      },
      "near_optimal": {
        "mean": 0.8334710743801652,
        "sd": 0.01664794442821267,
        "ci95": 0.020667839349138354
      },
      "unique": {
        "mean": 2.0113636363636362,
        "sd": 0.1041494476126327,
        "ci95": 0.1292978878468356
      }
    },
    "oracle_grid": {
      "hypervolume": {
        "mean": 0.3547658201014365,
        "sd": 0.0,
        "ci95": 0.0
      },
      "mip": {
        "mean": 0.5879172440154999,
        "sd": 0.0,
        "ci95": 0.0
      },
      "preference_regret": {
        "mean": 0.0,
        "sd": 0.0,
        "ci95": 0.0
      },
      "near_optimal": {
        "mean": 1.0,
        "sd": 0.0,
        "ci95": 0.0
      },
      "unique": {
        "mean": 1.9772727272727273,
        "sd": 0.0,
        "ci95": 0.0
      }
    }
  },
  "by_direction": {
    "up_up": {
      "sft": {
        "hypervolume": {
          "mean": 0.5547866157591552,
          "sd": 0.004451111549643458,
          "ci95": 0.005525898937842734
        },
        "mip": {
          "mean": 0.7173941342744271,
          "sd": 0.006962489498633501,
          "ci95": 0.008643686615385294
        }
      },
      "random": {
        "hypervolume": {
          "mean": 0.5300234743935974,
          "sd": 0.008807646321860372,
          "ci95": 0.010934384122267451
        },
        "mip": {
          "mean": 0.62319888712802,
          "sd": 0.00339621218061192,
          "ci95": 0.0042162783547932525
        }
      },
      "greedy_verifier": {
        "hypervolume": {
          "mean": 0.5415841777575114,
          "sd": 0.008233289246674151,
          "ci95": 0.010221339949746493
        },
        "mip": {
          "mean": 0.7101234331357681,
          "sd": 0.004102313866773968,
          "ci95": 0.005092878843020596
        }
      },
      "oracle_grid": {
        "hypervolume": {
          "mean": 0.5718122839434651,
          "sd": 0.0,
          "ci95": 0.0
        },
        "mip": {
          "mean": 0.7311801380469454,
          "sd": 0.0,
          "ci95": 0.0
        }
      },
      "sft_minus_greedy": {
        "hypervolume": {
          "mean": 0.013202438001643802,
          "sd": 0.011560547874820054,
          "ci95": 0.01435201488658834
        },
        "mip": {
          "mean": 0.0072707011386590635,
          "sd": 0.009143332516037751,
          "ci95": 0.0113511267635526
        }
      }
    },
    "up_down": {
      "sft": {
        "hypervolume": {
          "mean": 0.26637109162862416,
          "sd": 0.008435619826089943,
          "ci95": 0.010472526270605242
        },
        "mip": {
          "mean": 0.5324022327044661,
          "sd": 0.004339669649482129,
          "ci95": 0.005387547725821998
        }
      },
      "random": {
        "hypervolume": {
          "mean": 0.2768259355647035,
          "sd": 0.0035200122661316934,
          "ci95": 0.004369971820672209
        },
        "mip": {
          "mean": 0.4991439280925797,
          "sd": 0.0030486857043436617,
          "ci95": 0.0037848364183994494
        }
      },
      "greedy_verifier": {
        "hypervolume": {
          "mean": 0.2812522324658844,
          "sd": 0.003391659625414806,
          "ci95": 0.004210626517123577
        },
        "mip": {
          "mean": 0.5487054102595932,
          "sd": 0.0018000664676603043,
          "ci95": 0.0022347194112641747
        }
      },
      "oracle_grid": {
        "hypervolume": {
          "mean": 0.29986543339069294,
          "sd": 0.0,
          "ci95": 0.0
        },
        "mip": {
          "mean": 0.5622086416925245,
          "sd": 0.0,
          "ci95": 0.0
        }
      },
      "sft_minus_greedy": {
        "hypervolume": {
          "mean": -0.014881140837260253,
          "sd": 0.0057456736527104965,
          "ci95": 0.007133052402887352
        },
        "mip": {
          "mean": -0.016303177555127068,
          "sd": 0.0043833494770607615,
          "ci95": 0.005441774700394509
        }
      }
    },
    "down_up": {
      "sft": {
        "hypervolume": {
          "mean": 0.26823553613576073,
          "sd": 0.0136372196937986,
          "ci95": 0.01693013014403694
        },
        "mip": {
          "mean": 0.5337835686465041,
          "sd": 0.01378559041706564,
          "ci95": 0.01711432719525979
        }
      },
      "random": {
        "hypervolume": {
          "mean": 0.273847825696121,
          "sd": 0.0032118959376035605,
          "ci95": 0.003987456201021654
        },
        "mip": {
          "mean": 0.5008560719074203,
          "sd": 0.0030486857043436825,
          "ci95": 0.003784836418399475
        }
      },
      "greedy_verifier": {
        "hypervolume": {
          "mean": 0.28263117534455195,
          "sd": 0.007162604591140678,
          "ci95": 0.008892122486919516
        },
        "mip": {
          "mean": 0.5490746269413723,
          "sd": 0.0033286869480678356,
          "ci95": 0.004132448145949615
        }
      },
      "oracle_grid": {
        "hypervolume": {
          "mean": 0.2864046084806097,
          "sd": 0.0,
          "ci95": 0.0
        },
        "mip": {
          "mean": 0.5687956961246301,
          "sd": 0.0,
          "ci95": 0.0
        }
      },
      "sft_minus_greedy": {
        "hypervolume": {
          "mean": -0.014395639208791217,
          "sd": 0.01697110797142767,
          "ci95": 0.021069035558283978
        },
        "mip": {
          "mean": -0.015291058294868076,
          "sd": 0.014647717733847185,
          "ci95": 0.01818462803381549
        }
      }
    },
    "down_down": {
      "sft": {
        "hypervolume": {
          "mean": 0.21587562005975972,
          "sd": 0.011945547963630178,
          "ci95": 0.014829978999169532
        },
        "mip": {
          "mean": 0.43344417733066765,
          "sd": 0.015954219291847732,
          "ci95": 0.019806603913576
        }
      },
      "random": {
        "hypervolume": {
          "mean": 0.23123923655455006,
          "sd": 0.00532492595981477,
          "ci95": 0.0066107088931052806
        },
        "mip": {
          "mean": 0.3768011128719801,
          "sd": 0.003396212180611927,
          "ci95": 0.004216278354793261
        }
      },
      "greedy_verifier": {
        "hypervolume": {
          "mean": 0.24446755664926903,
          "sd": 0.002140164385024888,
          "ci95": 0.0026569390522161113
        },
        "mip": {
          "mean": 0.46594663489918126,
          "sd": 0.0020302006151077918,
          "ci95": 0.0025204228870719827
        }
      },
      "oracle_grid": {
        "hypervolume": {
          "mean": 0.2609809545909782,
          "sd": 0.0,
          "ci95": 0.0
        },
        "mip": {
          "mean": 0.48948450019789985,
          "sd": 0.0,
          "ci95": 0.0
        }
      },
      "sft_minus_greedy": {
        "hypervolume": {
          "mean": -0.028591936589509294,
          "sd": 0.012851095799986063,
          "ci95": 0.01595418489050146
        },
        "mip": {
          "mean": -0.032502457568513596,
          "sd": 0.016879600284616415,
          "ci95": 0.020955431973265923
        }
      }
    }
  },
  "sft_minus": {
    "random": {
      "hypervolume": {
        "mean": -0.0016669021564180487,
        "sd": 0.010411980805258838,
        "ci95": 0.012926109137217073
      },
      "mip": {
        "mean": 0.05425602823901628,
        "sd": 0.00958850553545619,
        "ci95": 0.01190379345988773
      },
      "preference_regret": {
        "mean": -0.054256028239016284,
        "sd": 0.00958850553545623,
        "ci95": 0.01190379345988778
      },
      "near_optimal": {
        "mean": 0.3338842975206612,
        "sd": 0.07947633038235787,
        "ci95": 0.09866707781760457
      },
      "unique": {
        "mean": -2.3181818181818183,
        "sd": 0.2821524776803439,
        "ci95": 0.35028240908687147
      }
    },
    "greedy_verifier": {
      "hypervolume": {
        "mean": -0.011166569658479253,
        "sd": 0.008460070908431904,
        "ci95": 0.010502881432104929
      },
      "mip": {
        "mean": -0.014206498069962437,
        "sd": 0.010572809308625373,
        "ci95": 0.013125772085677476
      },
      "preference_regret": {
        "mean": 0.014206498069962404,
        "sd": 0.010572809308625449,
        "ci95": 0.013125772085677572
      },
      "near_optimal": {
        "mean": -0.05123966942148759,
        "sd": 0.0774607415697123,
        "ci95": 0.09616479497101585
      },
      "unique": {
        "mean": 0.0795454545454544,
        "sd": 0.11220691851211084,
        "ci95": 0.1393009554825347
      }
    }
  },
  "sft_relative_percent": {
    "random": {
      "hypervolume": {
        "mean": -0.4802110660459289,
        "sd": 3.1398164434122986,
        "ci95": 3.897972036010412
      },
      "mip": {
        "mean": 10.851205647803255,
        "sd": 1.9177011070912375,
        "ci95": 2.3807586919775456
      }
    },
    "greedy_verifier": {
      "hypervolume": {
        "mean": -3.298275440887257,
        "sd": 2.4788612860440433,
        "ci95": 3.0774193804932795
      },
      "mip": {
        "mean": -2.496162558744969,
        "sd": 1.8505323175982593,
        "ci95": 2.2973709946853575
      }
    }
  }
}