Add files using upload-large-folder tool

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9969230769230769,
+    "total_flos": 82980012523520.0,
+    "train_loss": 1.2597844002178176,
+    "train_runtime": 2824.655,
+    "train_samples": 31200,
+    "train_samples_per_second": 11.046,
+    "train_steps_per_second": 0.086
+}

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "/data/cl/scratch/model_weights/gemma-2-9b-it",
+  "architectures": [
+    "Gemma2ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": 50.0,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "eos_token_id": 1,
+  "final_logit_softcapping": 30.0,
+  "head_dim": 256,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "model_type": "gemma2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 42,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "sliding_window_size": 4096,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "vocab_size": 256000
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.47.1"
+}

model.safetensors.index.json

@@ -0,0 +1,471 @@
+{
+  "metadata": {
+    "total_size": 18483411968
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

results_v2/flight_7features_1000ex_random_seed0-00003-of-00005/metrics.json

@@ -0,0 +1,793 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "bags",
+      "departure_time",
+      "layover_duration",
+      "number_of_stops",
+      "duration",
+      "flexibility",
+      "arrival_time"
+    ],
+    "reward_choices": {
+      "bags": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "layover_duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "flexibility": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "arrival_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "data/eval_data/flight_7features_1000ex_random_seed0.jsonl",
+    "random_set_file": "data/eval_data/flight_7features_1000ex_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.38,
+      0.36,
+      0.415,
+      0.43,
+      0.48
+    ],
+    "model_accs": [
+      0.38,
+      0.36,
+      0.415,
+      0.43,
+      0.48
+    ],
+    "model_reward_accs": [
+      0.35,
+      0.34,
+      0.405,
+      0.395,
+      0.49
+    ],
+    "model_consistencies": [
+      0.385,
+      0.61,
+      0.635,
+      0.555,
+      0.58
+    ],
+    "model_l2s": [
+      1.3847892809808289,
+      1.2680048798948134,
+      1.2503476686322863,
+      1.219235969399144,
+      1.1801551433535844,
+      1.1848773926055005
+    ],
+    "model_random_set_accs": [
+      0.3381500000000001,
+      0.4062500000000003,
+      0.41674999999999973,
+      0.4240500000000001,
+      0.44914999999999994,
+      0.44425000000000003
+    ],
+    "model_random_set_idxs": [
+      1.1033333333333333,
+      1.1825,
+      0.95,
+      1.3308333333333333,
+      0.555,
+      1.0058333333333334,
+      0.7825,
+      1.26,
+      0.73,
+      0.6725,
+      1.1916666666666667,
+      1.075,
+      1.0725,
+      1.3641666666666667,
+      0.74,
+      1.055,
+      1.0316666666666667,
+      1.2275,
+      1.1316666666666666,
+      0.9866666666666667,
+      0.685,
+      0.9058333333333334,
+      1.0216666666666667,
+      0.9458333333333333,
+      1.3466666666666667,
+      0.865,
+      1.3225,
+      1.1441666666666668,
+      0.9833333333333333,
+      1.0891666666666666,
+      1.3716666666666666,
+      1.0275,
+      0.9641666666666666,
+      0.9958333333333333,
+      1.2425,
+      1.1591666666666667,
+      0.92,
+      1.3433333333333333,
+      1.17,
+      0.845,
+      1.0833333333333333,
+      0.9916666666666667,
+      0.8816666666666667,
+      1.0041666666666667,
+      0.9916666666666667,
+      1.2233333333333334,
+      1.57,
+      0.8125,
+      0.8458333333333333,
+      1.3883333333333334,
+      1.1475,
+      1.1625,
+      0.8116666666666666,
+      1.0866666666666667,
+      0.8416666666666667,
+      1.12,
+      1.2033333333333334,
+      1.2141666666666666,
+      0.8991666666666667,
+      1.2108333333333334,
+      0.8833333333333333,
+      0.8733333333333333,
+      0.6175,
+      1.0683333333333334,
+      0.8675,
+      0.7241666666666666,
+      0.9633333333333334,
+      0.9675,
+      0.9191666666666667,
+      0.7975,
+      0.8508333333333333,
+      0.8958333333333334,
+      0.8458333333333333,
+      1.0891666666666666,
+      0.8283333333333334,
+      1.1033333333333333,
+      1.0333333333333334,
+      0.7075,
+      0.8383333333333334,
+      1.1275,
+      1.1275,
+      0.8875,
+      1.15,
+      0.9141666666666667,
+      0.825,
+      1.1325,
+      1.045,
+      1.1433333333333333,
+      1.1025,
+      0.6866666666666666,
+      1.0675,
+      0.8775,
+      1.24,
+      1.1541666666666666,
+      1.3358333333333334,
+      1.1783333333333332,
+      0.8775,
+      1.1125,
+      1.3433333333333333,
+      0.7241666666666666
+    ],
+    "model_probs": [
+      1.0682004425097703e-05,
+      1.6155276047273016e-05,
+      1.7661762029433023e-05,
+      2.2050915561682652e-05,
+      4.145994203964751e-05,
+      3.3719754736106986e-05
+    ],
+    "model_info_gains": [
+      -0.11226167028151596,
+      -0.14466060356320765,
+      0.18311620759922753,
+      0.10925975783983993,
+      -0.17316817087179692
+    ],
+    "model_delta_probs": [
+      5.473271622175313e-06,
+      1.506485982160008e-06,
+      4.38915353224963e-06,
+      1.940902647796486e-05,
+      -7.740187303540528e-06
+    ],
+    "model_expected_reward_accs": [
+      0.36,
+      0.375,
+      0.44,
+      0.445,
+      0.5
+    ],
+    "model_expected_consistencies": [
+      0.39,
+      0.64,
+      0.7,
+      0.67,
+      0.67
+    ],
+    "model_expected_l2s": [
+      1.3887765646104224,
+      1.2609800377789946,
+      1.2269848802079628,
+      1.1915773737897746,
+      1.159881637891288,
+      1.1476071344605352
+    ],
+    "model_expected_random_set_accs": [
+      0.3369500000000001,
+      0.40984999999999994,
+      0.43459999999999993,
+      0.44280000000000014,
+      0.4628500000000001,
+      0.4655500000000003
+    ],
+    "model_expected_random_set_idxs": [
+      1.125,
+      1.1975,
+      0.9166666666666666,
+      1.4183333333333332,
+      0.505,
+      1.0333333333333334,
+      0.8416666666666667,
+      1.2816666666666667,
+      0.7108333333333333,
+      0.7258333333333333,
+      1.1383333333333334,
+      1.1166666666666667,
+      1.1808333333333334,
+      1.1658333333333333,
+      1.0925,
+      1.0533333333333332,
+      0.8075,
+      1.2283333333333333,
+      1.1283333333333334,
+      0.995,
+      0.7466666666666667,
+      0.8566666666666667,
+      1.0208333333333333,
+      0.9575,
+      1.2675,
+      0.7883333333333333,
+      1.2816666666666667,
+      1.145,
+      1.0016666666666667,
+      1.0916666666666666,
+      1.365,
+      1.0091666666666668,
+      0.9033333333333333,
+      1.04,
+      1.2575,
+      1.1716666666666666,
+      0.8825,
+      1.3041666666666667,
+      1.1758333333333333,
+      0.8541666666666666,
+      1.0558333333333334,
+      1.0441666666666667,
+      0.8616666666666667,
+      0.95,
+      0.9533333333333334,
+      1.2075,
+      1.1958333333333333,
+      0.7516666666666667,
+      0.9266666666666666,
+      1.4733333333333334,
+      1.2025,
+      1.215,
+      0.8308333333333333,
+      1.3008333333333333,
+      0.8,
+      1.1066666666666667,
+      1.1925,
+      1.2375,
+      0.92,
+      1.24,
+      0.8808333333333334,
+      0.8583333333333333,
+      0.6341666666666667,
+      1.0775,
+      0.8933333333333333,
+      0.7175,
+      0.9216666666666666,
+      1.0433333333333332,
+      0.8908333333333334,
+      0.8108333333333333,
+      0.8133333333333334,
+      0.9075,
+      0.7808333333333334,
+      1.0366666666666666,
+      0.805,
+      1.1633333333333333,
+      1.1116666666666666,
+      0.7825,
+      0.8641666666666666,
+      1.0591666666666666,
+      1.165,
+      0.9,
+      1.1458333333333333,
+      0.9108333333333334,
+      0.7916666666666666,
+      1.1208333333333333,
+      1.0525,
+      1.1516666666666666,
+      0.9866666666666667,
+      0.69,
+      0.8216666666666667,
+      0.8041666666666667,
+      1.2908333333333333,
+      1.1116666666666666,
+      1.3716666666666666,
+      0.925,
+      0.9175,
+      1.1433333333333333,
+      1.39,
+      0.7816666666666666
+    ],
+    "infer_reward_accs": [
+      0.365,
+      0.425,
+      0.465,
+      0.475,
+      0.54
+    ],
+    "infer_l2s": [
+      1.3847892809808289,
+      1.287678892328789,
+      1.185999773927313,
+      1.093775097457443,
+      0.9950103856719794,
+      0.9019569300493474
+    ],
+    "infer_random_set_accs": [
+      0.3385,
+      0.3941000000000001,
+      0.4527000000000001,
+      0.49434999999999996,
+      0.5417000000000001,
+      0.5879499999999998
+    ],
+    "infer_random_set_idxs": [
+      0.845,
+      0.7375,
+      0.9166666666666666,
+      0.8308333333333333,
+      0.5316666666666666,
+      1.09,
+      0.4025,
+      1.3291666666666666,
+      1.1233333333333333,
+      1.1366666666666667,
+      1.5383333333333333,
+      1.5258333333333334,
+      1.3516666666666666,
+      1.2566666666666666,
+      0.7333333333333333,
+      0.9075,
+      0.6508333333333334,
+      1.3575,
+      0.91,
+      0.8133333333333334,
+      0.6066666666666667,
+      1.2166666666666666,
+      0.4116666666666667,
+      0.4683333333333333,
+      1.6958333333333333,
+      0.7491666666666666,
+      1.3258333333333334,
+      1.6158333333333332,
+      0.9233333333333333,
+      1.2941666666666667,
+      1.6108333333333333,
+      1.1508333333333334,
+      0.72,
+      1.3941666666666668,
+      1.7116666666666667,
+      1.3508333333333333,
+      0.8941666666666667,
+      1.5808333333333333,
+      0.7858333333333334,
+      1.0791666666666666,
+      0.9225,
+      1.2241666666666666,
+      0.9633333333333334,
+      1.0375,
+      0.3475,
+      0.9516666666666667,
+      1.3066666666666666,
+      1.2616666666666667,
+      0.235,
+      1.565,
+      0.9233333333333333,
+      0.8908333333333334,
+      1.1116666666666666,
+      1.2308333333333332,
+      0.6666666666666666,
+      1.4208333333333334,
+      0.8483333333333334,
+      0.965,
+      1.22,
+      1.3983333333333334,
+      0.37833333333333335,
+      0.6716666666666666,
+      0.6416666666666667,
+      1.6608333333333334,
+      1.0775,
+      0.6233333333333333,
+      1.0491666666666666,
+      0.6216666666666667,
+      0.9641666666666666,
+      0.39166666666666666,
+      0.7975,
+      1.1,
+      0.9625,
+      1.285,
+      0.7733333333333333,
+      0.8191666666666667,
+      0.8666666666666667,
+      0.3516666666666667,
+      0.3675,
+      0.9716666666666667,
+      1.4858333333333333,
+      0.4041666666666667,
+      1.4616666666666667,
+      0.9766666666666667,
+      0.9566666666666667,
+      1.2783333333333333,
+      1.3558333333333332,
+      1.7041666666666666,
+      0.7575,
+      0.715,
+      1.365,
+      0.5125,
+      1.4166666666666667,
+      0.9275,
+      1.2658333333333334,
+      0.8825,
+      0.30833333333333335,
+      1.43,
+      1.6833333333333333,
+      0.9216666666666666
+    ],
+    "infer_probs": [
+      1.0682004425097703e-05,
+      3.168546929970262e-05,
+      9.082096730821831e-05,
+      0.00030066073182908324,
+      0.0007775367784502407,
+      0.002964457072090454
+    ],
+    "infer_info_gains": [
+      0.8406699547937448,
+      0.8885070857772164,
+      0.8875210827458647,
+      0.8029342044415287,
+      0.7562832274623332
+    ],
+    "infer_delta_probs": [
+      2.1003464874604915e-05,
+      5.9135498008515694e-05,
+      0.00020983976452086496,
+      0.00047687604662115745,
+      0.002186920293640213
+    ],
+    "infer_expected_reward_accs": [
+      0.36,
+      0.5,
+      0.585,
+      0.6,
+      0.655
+    ],
+    "infer_expected_l2s": [
+      1.3887765646104224,
+      1.1164093878802575,
+      0.9681842359462353,
+      0.8565904229465078,
+      0.7569042958972374,
+      0.6780883204235127
+    ],
+    "infer_expected_random_set_accs": [
+      0.3369500000000001,
+      0.48695000000000005,
+      0.5604500000000004,
+      0.6121,
+      0.6576999999999998,
+      0.6933499999999997
+    ],
+    "infer_expected_random_set_idxs": [
+      0.8958333333333334,
+      0.8925,
+      0.9316666666666666,
+      0.9058333333333334,
+      0.6391666666666667,
+      1.1066666666666667,
+      0.6758333333333333,
+      1.2358333333333333,
+      1.0633333333333332,
+      1.0625,
+      1.3591666666666666,
+      1.3208333333333333,
+      1.1766666666666667,
+      1.1033333333333333,
+      1.1183333333333334,
+      1.0483333333333333,
+      0.5608333333333333,
+      1.2858333333333334,
+      0.9008333333333334,
+      0.8125,
+      0.7366666666666667,
+      1.1191666666666666,
+      0.6125,
+      0.6158333333333333,
+      1.515,
+      0.7825,
+      1.2466666666666666,
+      1.375,
+      0.935,
+      1.1716666666666666,
+      1.4433333333333334,
+      1.1308333333333334,
+      0.8141666666666667,
+      1.1808333333333334,
+      1.5008333333333332,
+      1.2641666666666667,
+      0.7875,
+      1.3591666666666666,
+      0.8908333333333334,
+      1.0825,
+      0.8966666666666666,
+      1.1641666666666666,
+      1.0025,
+      1.0283333333333333,
+      0.5541666666666667,
+      0.9975,
+      0.89,
+      1.0025,
+      0.5058333333333334,
+      1.42,
+      0.9475,
+      0.9675,
+      1.0725,
+      1.3008333333333333,
+      0.6966666666666667,
+      1.2066666666666668,
+      0.9275,
+      1.0216666666666667,
+      1.2058333333333333,
+      1.2541666666666667,
+      0.5966666666666667,
+      0.7883333333333333,
+      0.6975,
+      1.4041666666666666,
+      1.0566666666666666,
+      0.6616666666666666,
+      1.0408333333333333,
+      0.6933333333333334,
+      0.9425,
+      0.6391666666666667,
+      0.7758333333333334,
+      1.0475,
+      0.8725,
+      1.2325,
+      0.9158333333333334,
+      0.8691666666666666,
+      0.9183333333333333,
+      0.6166666666666667,
+      0.5725,
+      0.9766666666666667,
+      1.3541666666666667,
+      0.6883333333333334,
+      1.3758333333333332,
+      0.95,
+      0.8925,
+      1.0883333333333334,
+      1.1741666666666666,
+      1.44,
+      0.7983333333333333,
+      0.7408333333333333,
+      1.0025,
+      0.6175,
+      1.4058333333333333,
+      1.0175,
+      1.3366666666666667,
+      0.7083333333333334,
+      0.5641666666666667,
+      1.2675,
+      1.5625,
+      0.9358333333333333
+    ],
+    "kls": [
+      0.0,
+      2.764485415420897,
+      4.083612282988061,
+      4.853100073411079,
+      5.577996423073962,
+      6.299917905327967
+    ],
+    "model_random_set_direct_accs": [
+      0.36345,
+      0.4008000000000002,
+      0.4172500000000002,
+      0.42510000000000003,
+      0.43570000000000014,
+      0.4425500000000001
+    ],
+    "model_random_set_direct_idxs": [
+      1.2016666666666667,
+      1.6041666666666667,
+      1.3175,
+      1.4991666666666668,
+      1.0258333333333334,
+      1.6583333333333334,
+      1.2725,
+      1.3366666666666667,
+      0.9383333333333334,
+      1.1333333333333333,
+      1.2883333333333333,
+      1.1533333333333333,
+      1.1083333333333334,
+      1.4,
+      1.0608333333333333,
+      1.4016666666666666,
+      0.8816666666666667,
+      1.5866666666666667,
+      1.5983333333333334,
+      0.8758333333333334,
+      0.6883333333333334,
+      0.825,
+      1.5225,
+      1.2733333333333334,
+      1.4975,
+      1.7658333333333334,
+      1.4308333333333334,
+      0.9608333333333333,
+      1.0183333333333333,
+      1.4658333333333333,
+      1.5425,
+      1.2641666666666667,
+      1.47,
+      0.7941666666666667,
+      1.0416666666666667,
+      0.965,
+      1.49,
+      1.5225,
+      1.605,
+      1.2191666666666667,
+      1.2608333333333333,
+      0.6675,
+      1.3325,
+      1.01,
+      1.33,
+      1.5666666666666667,
+      1.28,
+      0.625,
+      0.9875,
+      1.5391666666666666,
+      1.3933333333333333,
+      1.6383333333333334,
+      1.3575,
+      1.2,
+      1.1683333333333332,
+      1.2491666666666668,
+      1.4091666666666667,
+      1.4408333333333334,
+      1.2916666666666667,
+      1.2558333333333334,
+      1.1541666666666666,
+      1.6608333333333334,
+      0.9733333333333334,
+      0.6458333333333334,
+      1.1725,
+      1.3008333333333333,
+      1.3325,
+      1.3116666666666668,
+      1.15,
+      1.43,
+      1.2775,
+      1.3741666666666668,
+      1.0808333333333333,
+      1.2275,
+      1.205,
+      1.4941666666666666,
+      1.2516666666666667,
+      1.4916666666666667,
+      1.4041666666666666,
+      1.1741666666666666,
+      1.6275,
+      1.37,
+      1.2358333333333333,
+      1.1116666666666666,
+      0.9825,
+      1.0125,
+      1.2633333333333334,
+      1.2283333333333333,
+      0.76,
+      0.7291666666666666,
+      0.9408333333333333,
+      1.3641666666666667,
+      1.5816666666666668,
+      1.3958333333333333,
+      1.6875,
+      1.3983333333333334,
+      1.4058333333333333,
+      1.6758333333333333,
+      1.5541666666666667,
+      1.2558333333333334
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3800000000000002,
+      0.6470999999999997,
+      0.6724000000000001,
+      0.6706999999999995,
+      0.6680999999999997,
+      0.6692999999999998
+    ],
+    "model_random_set_consistencies": [
+      0.3948000000000005,
+      0.59825,
+      0.6143000000000004,
+      0.61375,
+      0.6168,
+      0.62255
+    ]
+  }
+}

results_v2/flight_7features_1000ex_random_seed2-00002-of-00005/metrics.json

@@ -0,0 +1,793 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "bags",
+      "departure_time",
+      "layover_duration",
+      "number_of_stops",
+      "duration",
+      "flexibility",
+      "arrival_time"
+    ],
+    "reward_choices": {
+      "bags": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "layover_duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "flexibility": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "arrival_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "data/eval_data/flight_7features_1000ex_random_seed2.jsonl",
+    "random_set_file": "data/eval_data/flight_7features_1000ex_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.365,
+      0.455,
+      0.395,
+      0.42,
+      0.415
+    ],
+    "model_accs": [
+      0.365,
+      0.455,
+      0.395,
+      0.42,
+      0.415
+    ],
+    "model_reward_accs": [
+      0.315,
+      0.44,
+      0.395,
+      0.42,
+      0.395
+    ],
+    "model_consistencies": [
+      0.345,
+      0.645,
+      0.595,
+      0.58,
+      0.59
+    ],
+    "model_l2s": [
+      1.3700683272753613,
+      1.2944795316522721,
+      1.2327726653476123,
+      1.2217099849676234,
+      1.2139436572551319,
+      1.1868928204864002
+    ],
+    "model_random_set_accs": [
+      0.3405499999999999,
+      0.39030000000000004,
+      0.42299999999999954,
+      0.425,
+      0.42389999999999994,
+      0.4515
+    ],
+    "model_random_set_idxs": [
+      1.1441666666666668,
+      1.2066666666666668,
+      0.9616666666666667,
+      1.3483333333333334,
+      0.5675,
+      0.9841666666666666,
+      0.7875,
+      1.2383333333333333,
+      0.68,
+      0.6866666666666666,
+      1.1591666666666667,
+      1.1008333333333333,
+      1.0991666666666666,
+      1.3666666666666667,
+      0.7616666666666667,
+      1.0,
+      1.0208333333333333,
+      1.225,
+      1.1716666666666666,
+      1.0116666666666667,
+      0.6825,
+      0.8833333333333333,
+      1.0591666666666666,
+      1.0008333333333332,
+      1.3091666666666666,
+      0.8358333333333333,
+      1.3316666666666668,
+      1.1366666666666667,
+      1.0125,
+      1.0641666666666667,
+      1.3483333333333334,
+      0.9725,
+      1.0125,
+      0.965,
+      1.25,
+      1.1183333333333334,
+      0.9633333333333334,
+      1.3441666666666667,
+      1.1716666666666666,
+      0.8783333333333333,
+      1.1025,
+      1.0025,
+      0.8166666666666667,
+      1.0225,
+      1.035,
+      1.2833333333333334,
+      1.5816666666666668,
+      0.7641666666666667,
+      0.88,
+      1.4108333333333334,
+      1.1575,
+      1.1175,
+      0.7425,
+      1.11,
+      0.8008333333333333,
+      1.0883333333333334,
+      1.2491666666666668,
+      1.2775,
+      0.8291666666666667,
+      1.2166666666666666,
+      0.925,
+      0.9041666666666667,
+      0.63,
+      1.0225,
+      0.8266666666666667,
+      0.72,
+      0.9266666666666666,
+      1.0108333333333333,
+      0.97,
+      0.8116666666666666,
+      0.8983333333333333,
+      0.9216666666666666,
+      0.8308333333333333,
+      1.055,
+      0.8191666666666667,
+      1.15,
+      1.0333333333333334,
+      0.7216666666666667,
+      0.895,
+      1.1516666666666666,
+      1.0975,
+      0.8633333333333333,
+      1.1475,
+      0.9575,
+      0.8916666666666667,
+      1.165,
+      1.0641666666666667,
+      1.1716666666666666,
+      1.0808333333333333,
+      0.6991666666666667,
+      1.105,
+      0.8916666666666667,
+      1.26,
+      1.1941666666666666,
+      1.3325,
+      1.2108333333333334,
+      0.9166666666666666,
+      1.0733333333333333,
+      1.3008333333333333,
+      0.7566666666666667
+    ],
+    "model_probs": [
+      1.1530209140495628e-05,
+      3.496749090751469e-05,
+      0.0001308427111728288,
+      4.7090597271754005e-05,
+      3.97502925833787e-05,
+      8.153974053961607e-05
+    ],
+    "model_info_gains": [
+      -0.37833322748760595,
+      -0.07460540820225402,
+      -0.015029846597710601,
+      0.053987685017457716,
+      -0.024249671718773104
+    ],
+    "model_delta_probs": [
+      2.3437281767019062e-05,
+      9.587522026531408e-05,
+      -8.375211390107476e-05,
+      -7.340304688375303e-06,
+      4.178944795623737e-05
+    ],
+    "model_expected_reward_accs": [
+      0.3,
+      0.45,
+      0.43,
+      0.405,
+      0.44
+    ],
+    "model_expected_consistencies": [
+      0.34,
+      0.67,
+      0.63,
+      0.61,
+      0.65
+    ],
+    "model_expected_l2s": [
+      1.3740660101249462,
+      1.2741347348473824,
+      1.2174418592954424,
+      1.2037733181530863,
+      1.1780215135352248,
+      1.163510073631285
+    ],
+    "model_expected_random_set_accs": [
+      0.3388999999999999,
+      0.4064,
+      0.43355000000000005,
+      0.4348999999999996,
+      0.44875000000000015,
+      0.4614000000000002
+    ],
+    "model_expected_random_set_idxs": [
+      1.1466666666666667,
+      1.2166666666666666,
+      0.8916666666666667,
+      1.4383333333333332,
+      0.5416666666666666,
+      0.9916666666666667,
+      0.805,
+      1.2383333333333333,
+      0.65,
+      0.77,
+      1.1508333333333334,
+      1.1391666666666667,
+      1.1525,
+      1.1875,
+      1.13,
+      1.03,
+      0.8166666666666667,
+      1.2383333333333333,
+      1.2091666666666667,
+      1.0025,
+      0.7208333333333333,
+      0.855,
+      1.0891666666666666,
+      1.045,
+      1.255,
+      0.805,
+      1.2591666666666668,
+      1.15,
+      1.0283333333333333,
+      1.0708333333333333,
+      1.365,
+      0.9641666666666666,
+      0.935,
+      1.0033333333333334,
+      1.28,
+      1.1466666666666667,
+      0.9266666666666666,
+      1.33,
+      1.1875,
+      0.8533333333333334,
+      1.0791666666666666,
+      1.0283333333333333,
+      0.8475,
+      0.9675,
+      1.0091666666666668,
+      1.2441666666666666,
+      1.2425,
+      0.6683333333333333,
+      0.9533333333333334,
+      1.5166666666666666,
+      1.2575,
+      1.1658333333333333,
+      0.7658333333333334,
+      1.3033333333333332,
+      0.785,
+      1.1133333333333333,
+      1.255,
+      1.2591666666666668,
+      0.8641666666666666,
+      1.1991666666666667,
+      0.9425,
+      0.8908333333333334,
+      0.655,
+      1.0425,
+      0.8308333333333333,
+      0.6908333333333333,
+      0.88,
+      1.13,
+      0.94,
+      0.8058333333333333,
+      0.8541666666666666,
+      0.8941666666666667,
+      0.7416666666666667,
+      1.02,
+      0.7516666666666667,
+      1.2158333333333333,
+      1.0733333333333333,
+      0.7891666666666667,
+      0.925,
+      1.0633333333333332,
+      1.1433333333333333,
+      0.855,
+      1.1225,
+      0.9183333333333333,
+      0.845,
+      1.1416666666666666,
+      1.025,
+      1.18,
+      0.9558333333333333,
+      0.6975,
+      0.825,
+      0.835,
+      1.3191666666666666,
+      1.175,
+      1.3741666666666668,
+      0.9,
+      0.9291666666666667,
+      1.0841666666666667,
+      1.36,
+      0.7966666666666666
+    ],
+    "infer_reward_accs": [
+      0.32,
+      0.4,
+      0.455,
+      0.495,
+      0.545
+    ],
+    "infer_l2s": [
+      1.3700683272753613,
+      1.263540644616879,
+      1.1884149802702977,
+      1.0682631856347382,
+      0.9385821898824142,
+      0.8564917368460414
+    ],
+    "infer_random_set_accs": [
+      0.34009999999999985,
+      0.3968500000000001,
+      0.442,
+      0.50735,
+      0.5658499999999997,
+      0.6059999999999995
+    ],
+    "infer_random_set_idxs": [
+      0.815,
+      0.7591666666666667,
+      0.9233333333333333,
+      0.8183333333333334,
+      0.5908333333333333,
+      1.0841666666666667,
+      0.4225,
+      1.2591666666666668,
+      1.1133333333333333,
+      1.14,
+      1.5016666666666667,
+      1.5125,
+      1.3733333333333333,
+      1.2583333333333333,
+      0.7116666666666667,
+      0.9191666666666667,
+      0.6608333333333334,
+      1.3208333333333333,
+      0.8716666666666667,
+      0.7991666666666667,
+      0.6058333333333333,
+      1.2008333333333334,
+      0.43166666666666664,
+      0.44583333333333336,
+      1.6858333333333333,
+      0.8191666666666667,
+      1.2791666666666666,
+      1.5975,
+      0.875,
+      1.3175,
+      1.6516666666666666,
+      1.1433333333333333,
+      0.6958333333333333,
+      1.3291666666666666,
+      1.6758333333333333,
+      1.3233333333333333,
+      0.9133333333333333,
+      1.6166666666666667,
+      0.8208333333333333,
+      1.0108333333333333,
+      0.9041666666666667,
+      1.1966666666666668,
+      0.9566666666666667,
+      1.0358333333333334,
+      0.31666666666666665,
+      0.9641666666666666,
+      1.3,
+      1.2225,
+      0.22416666666666665,
+      1.5191666666666668,
+      0.9033333333333333,
+      0.9016666666666666,
+      1.1216666666666666,
+      1.2041666666666666,
+      0.665,
+      1.3816666666666666,
+      0.835,
+      1.01,
+      1.2175,
+      1.3966666666666667,
+      0.37333333333333335,
+      0.6691666666666667,
+      0.685,
+      1.7058333333333333,
+      1.09,
+      0.6075,
+      1.0441666666666667,
+      0.6233333333333333,
+      0.9833333333333333,
+      0.43666666666666665,
+      0.8216666666666667,
+      1.1375,
+      0.955,
+      1.3241666666666667,
+      0.7658333333333334,
+      0.81,
+      0.8616666666666667,
+      0.38333333333333336,
+      0.3725,
+      0.985,
+      1.5125,
+      0.39916666666666667,
+      1.3875,
+      0.9741666666666666,
+      0.9333333333333333,
+      1.2508333333333332,
+      1.3583333333333334,
+      1.6841666666666666,
+      0.7575,
+      0.7133333333333334,
+      1.3683333333333334,
+      0.5241666666666667,
+      1.3858333333333333,
+      0.9475,
+      1.3333333333333333,
+      0.8833333333333333,
+      0.2975,
+      1.3891666666666667,
+      1.7341666666666666,
+      0.9208333333333333
+    ],
+    "infer_probs": [
+      1.1530209140495628e-05,
+      3.6770383735245106e-05,
+      0.00011314616358674811,
+      0.0003607629427216352,
+      0.0010403553154208112,
+      0.002830883939193427
+    ],
+    "infer_info_gains": [
+      0.9010958600423478,
+      0.9592858913350802,
+      0.894395600031413,
+      0.8519376016415408,
+      0.7463549429258417
+    ],
+    "infer_delta_probs": [
+      2.5240174594749474e-05,
+      7.637577985150301e-05,
+      0.0002476167791348871,
+      0.0006795923726991759,
+      0.0017905286237726158
+    ],
+    "infer_expected_reward_accs": [
+      0.3,
+      0.455,
+      0.595,
+      0.56,
+      0.655
+    ],
+    "infer_expected_l2s": [
+      1.3740660101249462,
+      1.1175384430684387,
+      0.9614677317866236,
+      0.8440212068458718,
+      0.7432113750635955,
+      0.6675771315387196
+    ],
+    "infer_expected_random_set_accs": [
+      0.3388999999999999,
+      0.4811000000000002,
+      0.5496500000000001,
+      0.6141000000000001,
+      0.6609500000000004,
+      0.6969500000000001
+    ],
+    "infer_expected_random_set_idxs": [
+      0.91,
+      0.9225,
+      0.9491666666666667,
+      0.82,
+      0.7425,
+      1.0941666666666667,
+      0.645,
+      1.19,
+      1.0991666666666666,
+      1.11,
+      1.3291666666666666,
+      1.315,
+      1.2491666666666668,
+      1.0658333333333334,
+      1.125,
+      1.0175,
+      0.5691666666666667,
+      1.2875,
+      0.905,
+      0.8575,
+      0.6725,
+      1.0766666666666667,
+      0.6708333333333333,
+      0.61,
+      1.5541666666666667,
+      0.7716666666666666,
+      1.1833333333333333,
+      1.345,
+      0.9483333333333334,
+      1.1816666666666666,
+      1.4725,
+      1.0725,
+      0.7791666666666667,
+      1.1708333333333334,
+      1.46,
+      1.2316666666666667,
+      0.8408333333333333,
+      1.3975,
+      0.8625,
+      1.0491666666666666,
+      0.9183333333333333,
+      1.1416666666666666,
+      1.0,
+      1.0433333333333332,
+      0.5516666666666666,
+      1.03,
+      0.8533333333333334,
+      0.9283333333333333,
+      0.49666666666666665,
+      1.3933333333333333,
+      0.9208333333333333,
+      0.9741666666666666,
+      1.0516666666666667,
+      1.225,
+      0.6925,
+      1.2258333333333333,
+      0.9283333333333333,
+      1.01,
+      1.1566666666666667,
+      1.2733333333333334,
+      0.6208333333333333,
+      0.8175,
+      0.6883333333333334,
+      1.4491666666666667,
+      1.0783333333333334,
+      0.7141666666666666,
+      1.0275,
+      0.6533333333333333,
+      0.9825,
+      0.7033333333333334,
+      0.7991666666666667,
+      1.09,
+      0.9091666666666667,
+      1.2891666666666666,
+      0.9025,
+      0.8325,
+      0.8791666666666667,
+      0.6708333333333333,
+      0.5733333333333334,
+      1.03,
+      1.3566666666666667,
+      0.6966666666666667,
+      1.3325,
+      0.9975,
+      0.8975,
+      1.0841666666666667,
+      1.2008333333333334,
+      1.4625,
+      0.78,
+      0.7075,
+      1.0116666666666667,
+      0.6675,
+      1.4008333333333334,
+      1.0508333333333333,
+      1.3408333333333333,
+      0.6816666666666666,
+      0.5291666666666667,
+      1.3083333333333333,
+      1.5758333333333334,
+      0.9691666666666666
+    ],
+    "kls": [
+      0.0,
+      2.7614467737543786,
+      4.2902222118578015,
+      4.962432290756126,
+      5.803737064486347,
+      6.4852274265238545
+    ],
+    "model_random_set_direct_accs": [
+      0.3693499999999998,
+      0.4050499999999998,
+      0.42495,
+      0.43349999999999994,
+      0.4432999999999999,
+      0.44909999999999956
+    ],
+    "model_random_set_direct_idxs": [
+      1.235,
+      1.64,
+      1.3025,
+      1.54,
+      1.02,
+      1.5808333333333333,
+      1.2783333333333333,
+      1.275,
+      0.9025,
+      1.135,
+      1.2891666666666666,
+      1.1608333333333334,
+      1.1541666666666666,
+      1.4408333333333334,
+      1.0575,
+      1.2925,
+      0.9108333333333334,
+      1.565,
+      1.6825,
+      0.825,
+      0.6391666666666667,
+      0.6383333333333333,
+      1.5816666666666668,
+      1.3691666666666666,
+      1.4566666666666668,
+      1.7925,
+      1.4133333333333333,
+      0.9375,
+      1.0416666666666667,
+      1.435,
+      1.5458333333333334,
+      1.2408333333333332,
+      1.4741666666666666,
+      0.7966666666666666,
+      1.0533333333333332,
+      0.95,
+      1.52,
+      1.555,
+      1.6033333333333333,
+      1.1525,
+      1.2616666666666667,
+      0.6458333333333334,
+      1.2883333333333333,
+      1.01,
+      1.415,
+      1.5933333333333333,
+      1.3466666666666667,
+      0.5891666666666666,
+      1.035,
+      1.58,
+      1.4291666666666667,
+      1.6716666666666666,
+      1.305,
+      1.235,
+      1.1166666666666667,
+      1.3208333333333333,
+      1.4566666666666668,
+      1.4808333333333332,
+      1.2241666666666666,
+      1.265,
+      1.2341666666666666,
+      1.67,
+      0.9875,
+      0.6216666666666667,
+      1.1675,
+      1.2841666666666667,
+      1.3,
+      1.3933333333333333,
+      1.1841666666666666,
+      1.4475,
+      1.3083333333333333,
+      1.3741666666666668,
+      1.0291666666666666,
+      1.1875,
+      1.1416666666666666,
+      1.5208333333333333,
+      1.2266666666666666,
+      1.515,
+      1.395,
+      1.1391666666666667,
+      1.6116666666666666,
+      1.37,
+      1.1958333333333333,
+      1.1416666666666666,
+      1.0008333333333332,
+      1.0158333333333334,
+      1.2408333333333332,
+      1.2516666666666667,
+      0.7166666666666667,
+      0.7158333333333333,
+      0.9541666666666667,
+      1.345,
+      1.5733333333333333,
+      1.4183333333333332,
+      1.6675,
+      1.3516666666666666,
+      1.4133333333333333,
+      1.6308333333333334,
+      1.515,
+      1.2491666666666668
+    ],
+    "model_expected_random_set_consistencies": [
+      0.36999999999999966,
+      0.6500499999999998,
+      0.67125,
+      0.6772,
+      0.6798500000000004,
+      0.6848000000000005
+    ],
+    "model_random_set_consistencies": [
+      0.3848000000000003,
+      0.6016,
+      0.6188999999999999,
+      0.6209,
+      0.6182,
+      0.6229500000000001
+    ]
+  }
+}

results_v2/flight_noise1.0_random_seed1/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "data/eval_data/flight_noise1.0_random_seed1.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.3092948717948718,
+      0.33653846153846156,
+      0.358974358974359,
+      0.36538461538461536,
+      0.3942307692307692
+    ],
+    "model_accs": [
+      0.3092948717948718,
+      0.33653846153846156,
+      0.358974358974359,
+      0.36538461538461536,
+      0.3942307692307692
+    ],
+    "model_reward_accs": [
+      0.3108974358974359,
+      0.33814102564102566,
+      0.3541666666666667,
+      0.38782051282051283,
+      0.38782051282051283
+    ],
+    "model_consistencies": [
+      0.38621794871794873,
+      0.6602564102564102,
+      0.7003205128205128,
+      0.7083333333333334,
+      0.7131410256410257
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.488593391137741,
+      1.5120660225372382,
+      1.5256220267678013,
+      1.5701461090904691,
+      1.575628793589351
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.28886217948717946,
+      0.2733173076923079,
+      0.2638141025641026,
+      0.24469551282051297,
+      0.23775641025641042
+    ],
+    "model_random_set_idxs": [
+      0.7516025641025641,
+      1.2398504273504274,
+      1.2513354700854702,
+      1.077724358974359,
+      1.0416666666666667,
+      1.0958867521367521,
+      1.1466346153846154,
+      0.9738247863247863,
+      1.1132478632478633,
+      1.1252670940170941,
+      0.8223824786324786,
+      1.0395299145299146,
+      1.236912393162393,
+      0.8317307692307693,
+      1.049412393162393,
+      1.1463675213675213,
+      0.8792735042735043,
+      0.8685897435897436,
+      0.7107371794871795,
+      0.6821581196581197,
+      1.0966880341880343,
+      0.9917200854700855,
+      0.7256944444444444,
+      0.9105235042735043,
+      1.1626602564102564,
+      0.9802350427350427,
+      0.9564636752136753,
+      1.1682692307692308,
+      0.7441239316239316,
+      1.017628205128205,
+      1.0587606837606838,
+      1.0446047008547008,
+      0.8939636752136753,
+      0.9735576923076923,
+      0.9313568376068376,
+      0.8466880341880342,
+      0.5029380341880342,
+      0.9006410256410257,
+      1.1845619658119657,
+      1.001068376068376,
+      1.111378205128205,
+      0.9778311965811965,
+      0.6989850427350427,
+      0.9236111111111112,
+      0.9113247863247863,
+      0.6260683760683761,
+      0.7227564102564102,
+      1.3520299145299146,
+      1.4366987179487178,
+      1.076655982905983,
+      0.9262820512820513,
+      1.1776175213675213,
+      1.1904380341880343,
+      1.2529380341880343,
+      0.6541132478632479,
+      1.2911324786324787,
+      0.6514423076923077,
+      0.7948717948717948,
+      0.9807692307692307,
+      1.0916132478632479,
+      1.0221688034188035,
+      0.7678952991452992,
+      1.0101495726495726,
+      1.0643696581196582,
+      1.3848824786324787,
+      1.0972222222222223,
+      1.1314102564102564,
+      0.6602564102564102,
+      1.0998931623931625,
+      1.0881410256410255,
+      0.7264957264957265,
+      1.0109508547008548,
+      1.1669337606837606,
+      0.8792735042735043,
+      1.186965811965812,
+      1.0408653846153846,
+      1.0817307692307692,
+      1.1992521367521367,
+      1.0734508547008548,
+      1.201655982905983,
+      1.0200320512820513,
+      0.7930021367521367,
+      1.2112713675213675,
+      1.2564102564102564,
+      0.9415064102564102,
+      1.1185897435897436,
+      0.6917735042735043,
+      0.7462606837606838,
+      1.1081730769230769,
+      1.0283119658119657,
+      0.8859508547008547,
+      1.1773504273504274,
+      1.342681623931624,
+      0.9965277777777778,
+      0.9716880341880342,
+      1.0950854700854702,
+      0.8635149572649573,
+      0.8985042735042735,
+      0.7930021367521367,
+      0.6583867521367521
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.0008562752320548814,
+      0.000626074884244443,
+      0.0007189076201366114,
+      0.0005106474804269738,
+      0.00044727789637160074
+    ],
+    "model_info_gains": [
+      -1.4756715598303118,
+      -0.8239025986218158,
+      -0.198711474618471,
+      -0.32940323328514554,
+      -0.05291287352950853
+    ],
+    "model_delta_probs": [
+      -0.000743777266973702,
+      -0.00023020034781043842,
+      9.283273589216845e-05,
+      -0.0002082601397096375,
+      -6.336958405537311e-05
+    ],
+    "model_expected_reward_accs": [
+      0.3092948717948718,
+      0.3605769230769231,
+      0.36378205128205127,
+      0.3926282051282051,
+      0.40224358974358976
+    ],
+    "model_expected_consistencies": [
+      0.3814102564102564,
+      0.7275641025641025,
+      0.7628205128205128,
+      0.7307692307692307,
+      0.7628205128205128
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.506790719075094,
+      1.5360384522431048,
+      1.5547975304541466,
+      1.5919184441528786,
+      1.6028094695852926
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.2776923076923076,
+      0.2618910256410255,
+      0.25028846153846157,
+      0.2315064102564103,
+      0.22325320512820504
+    ],
+    "model_expected_random_set_idxs": [
+      0.9943910256410257,
+      1.2791132478632479,
+      1.217681623931624,
+      1.0237713675213675,
+      1.0221688034188035,
+      1.0809294871794872,
+      1.2069978632478633,
+      0.9225427350427351,
+      1.2305021367521367,
+      1.2871260683760684,
+      0.9276175213675214,
+      1.0584935897435896,
+      1.1647970085470085,
+      0.8776709401709402,
+      1.0456730769230769,
+      1.202190170940171,
+      0.9268162393162394,
+      0.8560363247863247,
+      0.6757478632478633,
+      0.7045940170940171,
+      1.1439636752136753,
+      1.0256410256410255,
+      0.7163461538461539,
+      0.9115918803418803,
+      1.1388888888888888,
+      0.9364316239316239,
+      0.9455128205128205,
+      1.1821581196581197,
+      0.7414529914529915,
+      1.0379273504273505,
+      1.0657051282051282,
+      1.0090811965811965,
+      0.936965811965812,
+      1.0384615384615385,
+      0.905982905982906,
+      0.9284188034188035,
+      0.6543803418803419,
+      0.8450854700854701,
+      1.2077991452991452,
+      0.9909188034188035,
+      1.0763888888888888,
+      0.9850427350427351,
+      0.750267094017094,
+      0.999732905982906,
+      0.9535256410256411,
+      0.6829594017094017,
+      0.7305021367521367,
+      1.2999465811965811,
+      1.3944978632478633,
+      1.0093482905982907,
+      0.9540598290598291,
+      1.138621794871795,
+      1.1354166666666667,
+      1.1933760683760684,
+      0.6626602564102564,
+      1.2443910256410255,
+      0.7307692307692307,
+      0.8004807692307693,
+      0.9572649572649573,
+      1.0996260683760684,
+      0.9869123931623932,
+      1.1316773504273505,
+      1.078258547008547,
+      1.107371794871795,
+      1.3247863247863247,
+      0.9460470085470085,
+      1.1111111111111112,
+      0.4791666666666667,
+      1.0819978632478633,
+      1.138621794871795,
+      0.7067307692307693,
+      1.0584935897435896,
+      1.1228632478632479,
+      0.9153311965811965,
+      1.1599893162393162,
+      1.001068376068376,
+      1.0924145299145298,
+      1.2446581196581197,
+      0.8814102564102564,
+      1.1276709401709402,
+      1.0643696581196582,
+      0.8010149572649573,
+      1.2411858974358974,
+      1.2240918803418803,
+      0.8993055555555556,
+      1.0544871794871795,
+      0.7083333333333334,
+      0.8245192307692307,
+      1.0916132478632479,
+      0.9535256410256411,
+      0.6589209401709402,
+      1.2075320512820513,
+      1.3098290598290598,
+      0.9719551282051282,
+      0.9588675213675214,
+      0.9754273504273504,
+      0.9489850427350427,
+      0.9415064102564102,
+      0.8522970085470085,
+      0.6522435897435898
+    ],
+    "infer_reward_accs": [
+      0.3141025641025641,
+      0.3269230769230769,
+      0.36378205128205127,
+      0.4182692307692308,
+      0.3733974358974359
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.4570254666752571,
+      1.5643271957761555,
+      1.5785840639023472,
+      1.5568861536038834,
+      1.4999420865856692
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.30150641025640995,
+      0.24293269230769268,
+      0.23554487179487169,
+      0.2507692307692312,
+      0.27657051282051276
+    ],
+    "infer_random_set_idxs": [
+      0.6968482905982906,
+      1.0243055555555556,
+      1.516292735042735,
+      1.1105769230769231,
+      1.0029380341880343,
+      1.3060897435897436,
+      1.2216880341880343,
+      0.8766025641025641,
+      1.1647970085470085,
+      1.1947115384615385,
+      0.6474358974358975,
+      1.3205128205128205,
+      1.2366452991452992,
+      0.5852029914529915,
+      0.8776709401709402,
+      1.1784188034188035,
+      1.139155982905983,
+      0.9639423076923077,
+      0.5128205128205128,
+      0.6188568376068376,
+      1.1957799145299146,
+      1.048878205128205,
+      0.5069444444444444,
+      0.6973824786324786,
+      0.9292200854700855,
+      1.1375534188034189,
+      0.7922008547008547,
+      1.1680021367521367,
+      0.8985042735042735,
+      0.7355769230769231,
+      1.093215811965812,
+      0.6776175213675214,
+      1.2457264957264957,
+      0.718215811965812,
+      0.9556623931623932,
+      0.6052350427350427,
+      0.49252136752136755,
+      0.624465811965812,
+      0.9209401709401709,
+      1.0552884615384615,
+      1.2280982905982907,
+      0.749465811965812,
+      0.6573183760683761,
+      0.6672008547008547,
+      0.6514423076923077,
+      0.5814636752136753,
+      0.8707264957264957,
+      1.1033653846153846,
+      1.4586004273504274,
+      1.1153846153846154,
+      0.5814636752136753,
+      0.8982371794871795,
+      1.452457264957265,
+      1.1655982905982907,
+      0.6696047008547008,
+      1.314102564102564,
+      0.5122863247863247,
+      0.8336004273504274,
+      0.8605769230769231,
+      0.9989316239316239,
+      0.8199786324786325,
+      0.7091346153846154,
+      1.109508547008547,
+      0.9294871794871795,
+      1.138087606837607,
+      0.9089209401709402,
+      0.905982905982906,
+      0.7176816239316239,
+      1.2230235042735043,
+      1.2831196581196582,
+      0.6706730769230769,
+      1.1047008547008548,
+      1.422275641025641,
+      0.9209401709401709,
+      1.3832799145299146,
+      1.1837606837606838,
+      1.077724358974359,
+      1.2553418803418803,
+      0.9861111111111112,
+      1.0333867521367521,
+      1.2566773504273505,
+      0.6842948717948718,
+      1.3541666666666667,
+      1.0024038461538463,
+      1.064102564102564,
+      1.2804487179487178,
+      0.6348824786324786,
+      0.8934294871794872,
+      1.3405448717948718,
+      0.9628739316239316,
+      0.8488247863247863,
+      1.2064636752136753,
+      1.5603632478632479,
+      0.8277243589743589,
+      0.7935363247863247,
+      1.126602564102564,
+      0.9636752136752137,
+      1.1479700854700854,
+      0.7793803418803419,
+      0.4479166666666667
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.0,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "infer_info_gains": [
+      -6.509661027808993,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "infer_delta_probs": [
+      -0.0016000524990285835,
+      0.0,
+      0.0,
+      0.0,
+      0.0
+    ],
+    "infer_expected_reward_accs": [
+      0.3092948717948718,
+      0.34935897435897434,
+      0.3814102564102564,
+      0.4407051282051282,
+      0.3942307692307692
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      1.548047939489473,
+      1.60560536490735,
+      1.5397731328120494,
+      1.426096590588756,
+      1.2828686132108222
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.25599358974358966,
+      0.2184294871794872,
+      0.21597756410256408,
+      0.2346955128205129,
+      0.2629807692307692
+    ],
+    "infer_expected_random_set_idxs": [
+      0.9559294871794872,
+      1.0616987179487178,
+      1.3971688034188035,
+      1.0726495726495726,
+      0.9887820512820513,
+      1.2305021367521367,
+      1.2008547008547008,
+      0.8691239316239316,
+      1.2200854700854702,
+      1.1955128205128205,
+      0.8178418803418803,
+      1.2168803418803418,
+      1.1663995726495726,
+      0.6968482905982906,
+      0.9444444444444444,
+      1.1439636752136753,
+      1.063034188034188,
+      1.0109508547008548,
+      0.6172542735042735,
+      0.7045940170940171,
+      1.140758547008547,
+      1.0050747863247864,
+      0.6268696581196581,
+      0.7868589743589743,
+      0.9732905982905983,
+      1.0352564102564104,
+      0.8293269230769231,
+      1.1204594017094016,
+      0.9118589743589743,
+      0.780715811965812,
+      1.125534188034188,
+      0.7636217948717948,
+      1.1669337606837606,
+      0.8210470085470085,
+      0.9548611111111112,
+      0.7828525641025641,
+      0.7267628205128205,
+      0.6955128205128205,
+      0.9674145299145299,
+      1.014155982905983,
+      1.0956196581196582,
+      0.7636217948717948,
+      0.7510683760683761,
+      0.8199786324786325,
+      0.7086004273504274,
+      0.7174145299145299,
+      0.9268162393162394,
+      1.0654380341880343,
+      1.3293269230769231,
+      1.0502136752136753,
+      0.6915064102564102,
+      0.9556623931623932,
+      1.3253205128205128,
+      1.1527777777777777,
+      0.7307692307692307,
+      1.1989850427350428,
+      0.6375534188034188,
+      0.8832799145299145,
+      0.8485576923076923,
+      1.0104166666666667,
+      0.8309294871794872,
+      1.1356837606837606,
+      1.0627670940170941,
+      0.9545940170940171,
+      1.0964209401709402,
+      0.8135683760683761,
+      0.9447115384615384,
+      0.6599893162393162,
+      1.2478632478632479,
+      1.2417200854700854,
+      0.7905982905982906,
+      1.0838675213675213,
+      1.3133012820512822,
+      0.9791666666666666,
+      1.265224358974359,
+      1.1613247863247864,
+      1.079594017094017,
+      1.2144764957264957,
+      0.84375,
+      0.9914529914529915,
+      1.186965811965812,
+      0.7128739316239316,
+      1.2719017094017093,
+      1.0197649572649572,
+      1.0857371794871795,
+      1.1589209401709402,
+      0.6957799145299145,
+      0.9516559829059829,
+      1.2593482905982907,
+      0.8496260683760684,
+      0.7083333333333334,
+      1.1901709401709402,
+      1.4030448717948718,
+      0.8325320512820513,
+      0.8322649572649573,
+      1.0480769230769231,
+      1.017094017094017,
+      1.1338141025641026,
+      0.8125,
+      0.5964209401709402
+    ],
+    "kls": [
+      0.0,
+      1.6624130876438907,
+      2.9342170279332183,
+      3.409500552850377,
+      3.1620787507705788,
+      2.4771226505112014
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.31410256410256376,
+      0.29105769230769224,
+      0.2821474358974359,
+      0.27145833333333336,
+      0.2632211538461537
+    ],
+    "model_random_set_direct_idxs": [
+      1.1461004273504274,
+      1.172275641025641,
+      1.3936965811965811,
+      1.2927350427350428,
+      1.0520833333333333,
+      0.6209935897435898,
+      0.7486645299145299,
+      0.7700320512820513,
+      1.0683760683760684,
+      1.515224358974359,
+      1.2780448717948718,
+      0.8314636752136753,
+      1.3645833333333333,
+      0.9244123931623932,
+      1.4043803418803418,
+      1.4383012820512822,
+      1.1690705128205128,
+      1.0088141025641026,
+      0.7243589743589743,
+      0.8830128205128205,
+      0.6164529914529915,
+      1.1207264957264957,
+      0.8178418803418803,
+      1.2443910256410255,
+      0.9278846153846154,
+      0.8926282051282052,
+      1.0331196581196582,
+      1.327190170940171,
+      0.7478632478632479,
+      1.2446581196581197,
+      1.376602564102564,
+      1.3116987179487178,
+      0.4097222222222222,
+      1.3044871794871795,
+      0.8888888888888888,
+      1.0846688034188035,
+      0.8360042735042735,
+      0.7889957264957265,
+      1.0397970085470085,
+      0.6957799145299145,
+      1.201655982905983,
+      1.282318376068376,
+      0.7313034188034188,
+      1.1487713675213675,
+      1.0841346153846154,
+      0.6156517094017094,
+      0.8026175213675214,
+      1.361912393162393,
+      1.3023504273504274,
+      1.0248397435897436,
+      1.2222222222222223,
+      0.8263888888888888,
+      1.2069978632478633,
+      1.311965811965812,
+      1.1204594017094016,
+      1.2126068376068375,
+      0.8552350427350427,
+      0.7417200854700855,
+      1.3514957264957266,
+      0.9887820512820513,
+      1.0547542735042734,
+      0.8034188034188035,
+      1.1442307692307692,
+      0.6533119658119658,
+      1.311965811965812,
+      0.9241452991452992,
+      1.0264423076923077,
+      0.453525641025641,
+      0.7932692307692307,
+      1.1159188034188035,
+      1.0333867521367521,
+      0.6984508547008547,
+      1.267628205128205,
+      0.8333333333333334,
+      1.1832264957264957,
+      0.7342414529914529,
+      0.7839209401709402,
+      1.0093482905982907,
+      0.8194444444444444,
+      1.141559829059829,
+      1.138087606837607,
+      0.7967414529914529,
+      1.015491452991453,
+      0.9989316239316239,
+      1.1861645299145298,
+      1.0852029914529915,
+      0.6704059829059829,
+      0.7948717948717948,
+      1.0819978632478633,
+      1.2299679487179487,
+      0.4780982905982906,
+      1.046741452991453,
+      1.3178418803418803,
+      1.1428952991452992,
+      1.3341346153846154,
+      0.6434294871794872,
+      1.0085470085470085,
+      0.46127136752136755,
+      0.4965277777777778,
+      0.7430555555555556
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7342788461538461,
+      0.73911858974359,
+      0.7412339743589745,
+      0.7370833333333332,
+      0.7419230769230766
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.6815384615384614,
+      0.6849839743589738,
+      0.6816506410256413,
+      0.6841987179487181,
+      0.6892307692307688
+    ]
+  }
+}

results_v2/flight_uniform_kmin_0_seed2/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_0_seed2.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.8221153846153846,
+      0.8910256410256411,
+      0.9262820512820513,
+      0.9375,
+      0.9423076923076923
+    ],
+    "model_accs": [
+      0.8221153846153846,
+      0.8910256410256411,
+      0.9262820512820513,
+      0.9375,
+      0.9423076923076923
+    ],
+    "model_reward_accs": [
+      0.5096153846153846,
+      0.9118589743589743,
+      0.9551282051282052,
+      0.9551282051282052,
+      0.969551282051282
+    ],
+    "model_consistencies": [
+      0.5064102564102564,
+      0.9262820512820513,
+      0.9455128205128205,
+      0.9503205128205128,
+      0.9551282051282052
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.0933534860512244,
+      1.0708080358699652,
+      1.065946616284595,
+      1.0451529059469347,
+      1.0601420222465345
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.48056089743589797,
+      0.4921634615384619,
+      0.49626602564102595,
+      0.5071474358974355,
+      0.49951923076923066
+    ],
+    "model_random_set_idxs": [
+      0.625267094017094,
+      1.267094017094017,
+      1.2377136752136753,
+      1.0769230769230769,
+      0.9748931623931624,
+      1.1474358974358974,
+      1.1971153846153846,
+      0.9505876068376068,
+      1.1209935897435896,
+      1.126068376068376,
+      0.8138354700854701,
+      1.0990918803418803,
+      1.1519764957264957,
+      0.8514957264957265,
+      1.0082799145299146,
+      1.174412393162393,
+      0.8782051282051282,
+      0.9425747863247863,
+      0.7198183760683761,
+      0.6501068376068376,
+      1.1423611111111112,
+      0.9722222222222222,
+      0.6818910256410257,
+      0.9420405982905983,
+      1.1995192307692308,
+      1.064102564102564,
+      0.9404380341880342,
+      1.232371794871795,
+      0.7147435897435898,
+      0.9564636752136753,
+      1.0558226495726495,
+      1.030181623931624,
+      0.9441773504273504,
+      0.9337606837606838,
+      0.9535256410256411,
+      0.7606837606837606,
+      0.47649572649572647,
+      0.7962072649572649,
+      1.1888354700854702,
+      0.9786324786324786,
+      1.155181623931624,
+      0.9623397435897436,
+      0.6947115384615384,
+      0.8453525641025641,
+      0.8894230769230769,
+      0.6001602564102564,
+      0.7096688034188035,
+      1.3659188034188035,
+      1.435897435897436,
+      1.0966880341880343,
+      0.8621794871794872,
+      1.21875,
+      1.2043269230769231,
+      1.2227564102564104,
+      0.6097756410256411,
+      1.345352564102564,
+      0.6148504273504274,
+      0.7743055555555556,
+      0.9014423076923077,
+      1.0365918803418803,
+      0.9802350427350427,
+      0.7072649572649573,
+      1.045405982905983,
+      1.0133547008547008,
+      1.3637820512820513,
+      1.110309829059829,
+      1.078525641025641,
+      0.6602564102564102,
+      1.106837606837607,
+      1.142628205128205,
+      0.7037927350427351,
+      1.0798611111111112,
+      1.1768162393162394,
+      0.8974358974358975,
+      1.1808226495726495,
+      1.0841346153846154,
+      1.0974893162393162,
+      1.1829594017094016,
+      1.0819978632478633,
+      1.2288995726495726,
+      0.9778311965811965,
+      0.8055555555555556,
+      1.2614850427350428,
+      1.2414529914529915,
+      0.9348290598290598,
+      1.1522435897435896,
+      0.6760149572649573,
+      0.7208867521367521,
+      1.126068376068376,
+      1.0235042735042734,
+      0.9121260683760684,
+      1.201655982905983,
+      1.3752670940170941,
+      0.9957264957264957,
+      1.0026709401709402,
+      1.0940170940170941,
+      0.9099893162393162,
+      0.9174679487179487,
+      0.8199786324786325,
+      0.6431623931623932
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.0033801424584055805,
+      0.003630901714407705,
+      0.0038165855123231916,
+      0.0035202413292625013,
+      0.0035379244659833075
+    ],
+    "model_info_gains": [
+      -0.10103101262341238,
+      -0.44363666662758944,
+      -0.2275962250096078,
+      -0.06779169682749006,
+      -0.10152091217003516
+    ],
+    "model_delta_probs": [
+      0.001780089959376997,
+      0.00025075925600212475,
+      0.00018568379791548641,
+      -0.00029634418306069023,
+      1.7683136720806148e-05
+    ],
+    "model_expected_reward_accs": [
+      0.4983974358974359,
+      0.9118589743589743,
+      0.9567307692307693,
+      0.9663461538461539,
+      0.9647435897435898
+    ],
+    "model_expected_consistencies": [
+      0.4951923076923077,
+      0.9262820512820513,
+      0.9503205128205128,
+      0.9583333333333334,
+      0.967948717948718
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.0741686577509684,
+      1.0513279621909053,
+      1.0416164743039567,
+      1.0322318677376976,
+      1.0309359099113542
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.4932371794871794,
+      0.5031089743589743,
+      0.5101121794871796,
+      0.5125000000000004,
+      0.5140705128205129
+    ],
+    "model_expected_random_set_idxs": [
+      0.8808760683760684,
+      1.3039529914529915,
+      1.1872329059829059,
+      1.0280448717948718,
+      1.0104166666666667,
+      1.1295405982905984,
+      1.281784188034188,
+      0.9166666666666666,
+      1.2406517094017093,
+      1.235042735042735,
+      0.969284188034188,
+      1.1132478632478633,
+      1.1378205128205128,
+      0.8880876068376068,
+      0.9855769230769231,
+      1.2053952991452992,
+      0.9158653846153846,
+      0.9375,
+      0.7403846153846154,
+      0.6928418803418803,
+      1.1997863247863247,
+      1.0336538461538463,
+      0.7048611111111112,
+      0.9225427350427351,
+      1.1826923076923077,
+      0.9914529914529915,
+      0.9545940170940171,
+      1.2502670940170941,
+      0.7099358974358975,
+      0.9834401709401709,
+      1.0555555555555556,
+      0.9612713675213675,
+      0.9770299145299145,
+      1.0347222222222223,
+      0.9155982905982906,
+      0.8036858974358975,
+      0.5670405982905983,
+      0.7801816239316239,
+      1.2102029914529915,
+      0.9874465811965812,
+      1.0852029914529915,
+      0.9244123931623932,
+      0.7345085470085471,
+      0.9201388888888888,
+      0.9137286324786325,
+      0.6466346153846154,
+      0.7110042735042735,
+      1.3173076923076923,
+      1.3651175213675213,
+      1.029647435897436,
+      0.875,
+      1.1957799145299146,
+      1.1618589743589745,
+      1.1824252136752136,
+      0.6073717948717948,
+      1.2705662393162394,
+      0.6765491452991453,
+      0.8277243589743589,
+      0.8790064102564102,
+      1.0974893162393162,
+      0.9802350427350427,
+      1.0924145299145298,
+      1.0964209401709402,
+      1.0400641025641026,
+      1.3202457264957266,
+      1.0074786324786325,
+      1.0739850427350428,
+      0.49278846153846156,
+      1.0568910256410255,
+      1.1736111111111112,
+      0.7120726495726496,
+      1.1220619658119657,
+      1.094818376068376,
+      0.905715811965812,
+      1.1853632478632479,
+      1.0592948717948718,
+      1.0769230769230769,
+      1.2849893162393162,
+      0.9139957264957265,
+      1.1423611111111112,
+      1.048344017094017,
+      0.7943376068376068,
+      1.2748397435897436,
+      1.2409188034188035,
+      0.8792735042735043,
+      1.1124465811965811,
+      0.6810897435897436,
+      0.7847222222222222,
+      1.1156517094017093,
+      0.8776709401709402,
+      0.6834935897435898,
+      1.235844017094017,
+      1.3611111111111112,
+      0.9821047008547008,
+      0.9655448717948718,
+      1.0299145299145298,
+      0.9284188034188035,
+      0.9698183760683761,
+      0.8514957264957265,
+      0.6279380341880342
+    ],
+    "infer_reward_accs": [
+      0.5096153846153846,
+      0.8862179487179487,
+      0.9375,
+      0.969551282051282,
+      0.9711538461538461
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.2207956383946548,
+      1.1863634375487317,
+      1.177472282284918,
+      1.1753979430360915,
+      1.1657526768202007
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.42116987179487253,
+      0.4382692307692311,
+      0.44387820512820564,
+      0.4456570512820514,
+      0.4488141025641031
+    ],
+    "infer_random_set_idxs": [
+      0.36030982905982906,
+      0.9049145299145299,
+      1.8667200854700854,
+      1.080662393162393,
+      1.044871794871795,
+      1.7564102564102564,
+      1.467147435897436,
+      0.9214743589743589,
+      1.298344017094017,
+      1.1129807692307692,
+      0.1391559829059829,
+      1.810897435897436,
+      1.2545405982905984,
+      0.22756410256410256,
+      0.5082799145299145,
+      1.1680021367521367,
+      1.2019230769230769,
+      0.9639423076923077,
+      0.2780448717948718,
+      0.24358974358974358,
+      1.438568376068376,
+      1.079059829059829,
+      0.15571581196581197,
+      0.30288461538461536,
+      0.7841880341880342,
+      1.3376068376068375,
+      0.4185363247863248,
+      1.1258012820512822,
+      1.0400641025641026,
+      0.2387820512820513,
+      0.9444444444444444,
+      0.11351495726495726,
+      1.6875,
+      0.22302350427350429,
+      1.061431623931624,
+      0.22302350427350429,
+      0.11084401709401709,
+      0.32077991452991456,
+      0.8143696581196581,
+      1.1388888888888888,
+      1.3635149572649572,
+      0.3250534188034188,
+      0.24893162393162394,
+      0.15571581196581197,
+      0.28739316239316237,
+      0.21314102564102563,
+      0.9292200854700855,
+      1.0261752136752136,
+      1.7094017094017093,
+      1.123397435897436,
+      0.09615384615384616,
+      0.750534188034188,
+      1.811431623931624,
+      1.0841346153846154,
+      0.45539529914529914,
+      1.578792735042735,
+      0.06997863247863248,
+      0.9019764957264957,
+      0.6028311965811965,
+      1.0077457264957266,
+      0.5504807692307693,
+      0.5413995726495726,
+      1.1639957264957266,
+      0.9139957264957265,
+      1.0050747863247864,
+      0.8087606837606838,
+      0.8533653846153846,
+      0.8386752136752137,
+      1.5285790598290598,
+      1.6063034188034189,
+      0.2390491452991453,
+      1.4169337606837606,
+      1.8338675213675213,
+      0.9401709401709402,
+      1.7884615384615385,
+      1.4543269230769231,
+      1.3998397435897436,
+      1.594818376068376,
+      0.8691239316239316,
+      0.9837072649572649,
+      1.6527777777777777,
+      0.3993055555555556,
+      1.7622863247863247,
+      0.9537927350427351,
+      1.0069444444444444,
+      1.608440170940171,
+      0.2604166666666667,
+      0.9463141025641025,
+      1.6832264957264957,
+      0.7825854700854701,
+      0.906784188034188,
+      1.5691773504273505,
+      1.9447115384615385,
+      0.4890491452991453,
+      0.3530982905982906,
+      1.1923076923076923,
+      0.9457799145299145,
+      1.6290064102564104,
+      0.7954059829059829,
+      0.171741452991453
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.003243782877657162,
+      0.00368008805487126,
+      0.003957483840465032,
+      0.0041270115504065145,
+      0.004254848972600018
+    ],
+    "infer_info_gains": [
+      0.7612321090766456,
+      0.13345376268072284,
+      0.071906107233214,
+      0.04450809828077134,
+      0.03197228617352304
+    ],
+    "infer_delta_probs": [
+      0.0016437303786285785,
+      0.00043630517721409816,
+      0.0002773957855937717,
+      0.0001695277099414829,
+      0.00012783742219350339
+    ],
+    "infer_expected_reward_accs": [
+      0.4983974358974359,
+      1.0,
+      1.0,
+      1.0,
+      1.0
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      1.0586625709193653,
+      1.0353026796182656,
+      1.020848286582152,
+      1.0155832185229112,
+      1.0130428388081538
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5010416666666664,
+      0.5131730769230768,
+      0.5203846153846151,
+      0.5213621794871793,
+      0.5224038461538459
+    ],
+    "infer_expected_random_set_idxs": [
+      0.7863247863247863,
+      0.9375,
+      1.6663995726495726,
+      1.1132478632478633,
+      1.0133547008547008,
+      1.5259081196581197,
+      1.2711004273504274,
+      0.8050213675213675,
+      1.2809829059829059,
+      1.3245192307692308,
+      0.6455662393162394,
+      1.4639423076923077,
+      1.219818376068376,
+      0.5056089743589743,
+      0.8723290598290598,
+      1.235042735042735,
+      1.0990918803418803,
+      1.1167200854700854,
+      0.44738247863247865,
+      0.5165598290598291,
+      1.2473290598290598,
+      0.9821047008547008,
+      0.35844017094017094,
+      0.6092414529914529,
+      0.7932692307692307,
+      1.1463675213675213,
+      0.7620192307692307,
+      1.2224893162393162,
+      0.9198717948717948,
+      0.5576923076923077,
+      1.1474358974358974,
+      0.44898504273504275,
+      1.4043803418803418,
+      0.6143162393162394,
+      0.9353632478632479,
+      0.5990918803418803,
+      0.6129807692307693,
+      0.5128205128205128,
+      0.8092948717948718,
+      1.0985576923076923,
+      1.1642628205128205,
+      0.4962606837606838,
+      0.46821581196581197,
+      0.6266025641025641,
+      0.46634615384615385,
+      0.5101495726495726,
+      1.0325854700854702,
+      0.9821047008547008,
+      1.486912393162393,
+      1.0109508547008548,
+      0.4500534188034188,
+      0.8087606837606838,
+      1.592147435897436,
+      1.155715811965812,
+      0.5876068376068376,
+      1.203525641025641,
+      0.3998397435897436,
+      0.8912927350427351,
+      0.7299679487179487,
+      1.0259081196581197,
+      0.65625,
+      1.1076388888888888,
+      1.2072649572649572,
+      0.9519230769230769,
+      1.0192307692307692,
+      0.843215811965812,
+      0.9294871794871795,
+      0.6594551282051282,
+      1.5096153846153846,
+      1.453525641025641,
+      0.6124465811965812,
+      1.174412393162393,
+      1.5064102564102564,
+      0.9316239316239316,
+      1.4901175213675213,
+      1.232371794871795,
+      1.1682692307692308,
+      1.3557692307692308,
+      0.8579059829059829,
+      0.8995726495726496,
+      1.3015491452991452,
+      0.49412393162393164,
+      1.4973290598290598,
+      0.9724893162393162,
+      1.1332799145299146,
+      1.1850961538461537,
+      0.46768162393162394,
+      1.0144230769230769,
+      1.358440170940171,
+      0.6885683760683761,
+      0.6912393162393162,
+      1.3979700854700854,
+      1.6509081196581197,
+      0.6316773504273504,
+      0.6231303418803419,
+      1.1063034188034189,
+      1.0299145299145298,
+      1.3838141025641026,
+      0.6856303418803419,
+      0.37606837606837606
+    ],
+    "kls": [
+      0.0,
+      1.4860784428777483,
+      1.9611116287155386,
+      2.161708642369367,
+      2.2453335038931725,
+      2.261130443289179
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.5007051282051288,
+      0.5092948717948717,
+      0.5154487179487185,
+      0.5208173076923077,
+      0.5242788461538458
+    ],
+    "model_random_set_direct_idxs": [
+      1.1463675213675213,
+      1.231837606837607,
+      1.469284188034188,
+      1.3149038461538463,
+      1.1036324786324787,
+      0.7972756410256411,
+      0.813034188034188,
+      0.8068910256410257,
+      1.1327457264957266,
+      1.5539529914529915,
+      1.3514957264957266,
+      0.8888888888888888,
+      1.3330662393162394,
+      0.8958333333333334,
+      1.3771367521367521,
+      1.4783653846153846,
+      1.1517094017094016,
+      1.0974893162393162,
+      0.813034188034188,
+      1.032852564102564,
+      0.7545405982905983,
+      1.1623931623931625,
+      0.8766025641025641,
+      1.3039529914529915,
+      1.0331196581196582,
+      0.9452457264957265,
+      1.0539529914529915,
+      1.4308226495726495,
+      0.7668269230769231,
+      1.2919337606837606,
+      1.3322649572649572,
+      1.2836538461538463,
+      0.4877136752136752,
+      1.3944978632478633,
+      0.8886217948717948,
+      0.9577991452991453,
+      0.8023504273504274,
+      0.8547008547008547,
+      1.1290064102564104,
+      0.7729700854700855,
+      1.2740384615384615,
+      1.267628205128205,
+      0.7983440170940171,
+      1.1535790598290598,
+      1.1153846153846154,
+      0.6063034188034188,
+      0.8688568376068376,
+      1.3175747863247864,
+      1.171207264957265,
+      1.0040064102564104,
+      1.1760149572649572,
+      0.8547008547008547,
+      1.2542735042735043,
+      1.2996794871794872,
+      1.0849358974358974,
+      1.2849893162393162,
+      0.8565705128205128,
+      0.8154380341880342,
+      1.2644230769230769,
+      1.0836004273504274,
+      1.0985576923076923,
+      0.7612179487179487,
+      1.2267628205128205,
+      0.6842948717948718,
+      1.3354700854700854,
+      0.9508547008547008,
+      1.063568376068376,
+      0.49225427350427353,
+      0.8904914529914529,
+      1.2315705128205128,
+      1.0224358974358974,
+      0.7548076923076923,
+      1.2243589743589745,
+      0.8221153846153846,
+      1.327457264957265,
+      0.7751068376068376,
+      0.7470619658119658,
+      1.0341880341880343,
+      0.937767094017094,
+      1.1778846153846154,
+      1.1356837606837606,
+      0.7876602564102564,
+      1.047809829059829,
+      1.0325854700854702,
+      1.173878205128205,
+      1.0654380341880343,
+      0.6514423076923077,
+      0.8579059829059829,
+      1.0202991452991452,
+      1.157852564102564,
+      0.5518162393162394,
+      1.1338141025641026,
+      1.374465811965812,
+      1.1875,
+      1.3133012820512822,
+      0.7123397435897436,
+      1.0411324786324787,
+      0.47088675213675213,
+      0.4778311965811966,
+      0.7211538461538461
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7148076923076927,
+      0.740993589743589,
+      0.7489583333333334,
+      0.7529647435897439,
+      0.7568269230769236
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.6593108974358973,
+      0.6859775641025635,
+      0.6983974358974359,
+      0.7019391025641024,
+      0.7012500000000005
+    ]
+  }
+}

results_v2/flight_uniform_kmin_19_seed2/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_19_seed2.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.7291666666666666,
+      0.8365384615384616,
+      0.8653846153846154,
+      0.875,
+      0.8733974358974359
+    ],
+    "model_accs": [
+      0.7291666666666666,
+      0.8365384615384616,
+      0.8653846153846154,
+      0.875,
+      0.8733974358974359
+    ],
+    "model_reward_accs": [
+      0.421474358974359,
+      0.7788461538461539,
+      0.8509615384615384,
+      0.8637820512820513,
+      0.8766025641025641
+    ],
+    "model_consistencies": [
+      0.3814102564102564,
+      0.8108974358974359,
+      0.8509615384615384,
+      0.8685897435897436,
+      0.8605769230769231
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.083715281731148,
+      0.9836720244546802,
+      0.9465607970247223,
+      0.9047283926970445,
+      0.8550439327302658
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.49366987179487226,
+      0.544455128205128,
+      0.5627884615384614,
+      0.5815064102564097,
+      0.6043589743589741
+    ],
+    "model_random_set_idxs": [
+      0.7086004273504274,
+      1.1856303418803418,
+      1.2951388888888888,
+      1.0702457264957266,
+      0.9799679487179487,
+      1.1279380341880343,
+      1.1519764957264957,
+      0.9393696581196581,
+      1.1183226495726495,
+      1.140758547008547,
+      0.7793803418803419,
+      1.0678418803418803,
+      1.1789529914529915,
+      0.7879273504273504,
+      1.0069444444444444,
+      1.1581196581196582,
+      0.9294871794871795,
+      0.9308226495726496,
+      0.6770833333333334,
+      0.6396901709401709,
+      1.092681623931624,
+      0.9676816239316239,
+      0.6594551282051282,
+      0.8806089743589743,
+      1.110042735042735,
+      1.061431623931624,
+      0.9244123931623932,
+      1.1917735042735043,
+      0.7748397435897436,
+      0.9177350427350427,
+      1.110844017094017,
+      0.9973290598290598,
+      0.9575320512820513,
+      0.8800747863247863,
+      0.9676816239316239,
+      0.7681623931623932,
+      0.5261752136752137,
+      0.813034188034188,
+      1.0737179487179487,
+      0.9909188034188035,
+      1.1335470085470085,
+      0.9516559829059829,
+      0.6856303418803419,
+      0.8394764957264957,
+      0.844017094017094,
+      0.6578525641025641,
+      0.7879273504273504,
+      1.3100961538461537,
+      1.4599358974358974,
+      1.1044337606837606,
+      0.8456196581196581,
+      1.1306089743589745,
+      1.2467948717948718,
+      1.25,
+      0.6618589743589743,
+      1.2748397435897436,
+      0.6306089743589743,
+      0.7572115384615384,
+      0.9364316239316239,
+      1.0347222222222223,
+      0.9676816239316239,
+      0.7398504273504274,
+      1.0405982905982907,
+      0.9599358974358975,
+      1.327724358974359,
+      1.0542200854700854,
+      1.0539529914529915,
+      0.6848290598290598,
+      1.1252670940170941,
+      1.1274038461538463,
+      0.7692307692307693,
+      1.0104166666666667,
+      1.235042735042735,
+      0.9134615384615384,
+      1.1955128205128205,
+      1.0857371794871795,
+      1.060897435897436,
+      1.1802884615384615,
+      1.0702457264957266,
+      1.1639957264957266,
+      1.047542735042735,
+      0.7892628205128205,
+      1.2230235042735043,
+      1.1995192307692308,
+      1.0197649572649572,
+      1.1682692307692308,
+      0.7102029914529915,
+      0.7719017094017094,
+      1.1754807692307692,
+      1.0339209401709402,
+      0.8835470085470085,
+      1.1418269230769231,
+      1.3688568376068375,
+      0.9348290598290598,
+      0.96875,
+      1.0990918803418803,
+      0.9115918803418803,
+      0.9556623931623932,
+      0.8031517094017094,
+      0.625
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.00288181327040964,
+      0.0042670421775265295,
+      0.004002978999201864,
+      0.005162124971595388,
+      0.00614114460372925
+    ],
+    "model_info_gains": [
+      0.0503393787093703,
+      -0.009340652478045306,
+      -0.09062685473345419,
+      0.13400206892491992,
+      0.10306053809316655
+    ],
+    "model_delta_probs": [
+      0.0012817607713810565,
+      0.0013852289071168893,
+      -0.00026406317832466513,
+      0.0011591459723935243,
+      0.0009790196321338622
+    ],
+    "model_expected_reward_accs": [
+      0.40865384615384615,
+      0.8317307692307693,
+      0.8814102564102564,
+      0.9150641025641025,
+      0.9230769230769231
+    ],
+    "model_expected_consistencies": [
+      0.3782051282051282,
+      0.8701923076923077,
+      0.8942307692307693,
+      0.8974358974358975,
+      0.8974358974358975
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.0458749504649583,
+      0.9538449431931705,
+      0.9162152067466623,
+      0.8720890337048248,
+      0.8230982219353211
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5107371794871802,
+      0.5592147435897438,
+      0.5780448717948716,
+      0.5984775641025639,
+      0.6200641025641019
+    ],
+    "model_expected_random_set_idxs": [
+      0.9260149572649573,
+      1.2379807692307692,
+      1.2524038461538463,
+      1.0523504273504274,
+      0.9909188034188035,
+      1.0710470085470085,
+      1.2246260683760684,
+      0.875,
+      1.2243589743589745,
+      1.2534722222222223,
+      0.8608440170940171,
+      1.109775641025641,
+      1.154647435897436,
+      0.8322649572649573,
+      1.0168269230769231,
+      1.1971153846153846,
+      0.9612713675213675,
+      0.906517094017094,
+      0.6575854700854701,
+      0.6594551282051282,
+      1.1439636752136753,
+      1.0336538461538463,
+      0.6271367521367521,
+      0.8848824786324786,
+      1.1290064102564104,
+      1.016559829059829,
+      0.9297542735042735,
+      1.201655982905983,
+      0.7513354700854701,
+      0.9375,
+      1.0916132478632479,
+      0.9727564102564102,
+      0.9951923076923077,
+      0.9626068376068376,
+      0.9292200854700855,
+      0.8522970085470085,
+      0.6225961538461539,
+      0.7831196581196581,
+      1.1295405982905984,
+      0.969284188034188,
+      1.0945512820512822,
+      0.9431089743589743,
+      0.7246260683760684,
+      0.9115918803418803,
+      0.8717948717948718,
+      0.6893696581196581,
+      0.7727029914529915,
+      1.3066239316239316,
+      1.4188034188034189,
+      1.0793269230769231,
+      0.874732905982906,
+      1.139155982905983,
+      1.1971153846153846,
+      1.204059829059829,
+      0.6482371794871795,
+      1.2612179487179487,
+      0.6845619658119658,
+      0.7395833333333334,
+      0.8629807692307693,
+      1.030181623931624,
+      0.9431089743589743,
+      1.1193910256410255,
+      1.0844017094017093,
+      1.0037393162393162,
+      1.2753739316239316,
+      0.9433760683760684,
+      1.0486111111111112,
+      0.5136217948717948,
+      1.094818376068376,
+      1.1672008547008548,
+      0.7398504273504274,
+      1.0568910256410255,
+      1.2267628205128205,
+      0.9078525641025641,
+      1.1677350427350428,
+      1.0665064102564104,
+      1.1004273504273505,
+      1.2361111111111112,
+      0.8912927350427351,
+      1.1613247863247864,
+      1.1060363247863247,
+      0.7895299145299145,
+      1.2446581196581197,
+      1.203258547008547,
+      0.9813034188034188,
+      1.1161858974358974,
+      0.7107371794871795,
+      0.7785790598290598,
+      1.1696047008547008,
+      0.9599358974358975,
+      0.6642628205128205,
+      1.1866987179487178,
+      1.3557692307692308,
+      0.9209401709401709,
+      0.9431089743589743,
+      1.014155982905983,
+      0.937767094017094,
+      1.0024038461538463,
+      0.8851495726495726,
+      0.6129807692307693
+    ],
+    "infer_reward_accs": [
+      0.4198717948717949,
+      0.6073717948717948,
+      0.7211538461538461,
+      0.8092948717948718,
+      0.8541666666666666
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.1593058959578144,
+      1.0470372240975332,
+      0.9850300585418577,
+      0.939190325334155,
+      0.89206678197499
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.4611698717948718,
+      0.5198076923076927,
+      0.5495192307692308,
+      0.5719391025641029,
+      0.5938942307692308
+    ],
+    "infer_random_set_idxs": [
+      0.3955662393162393,
+      0.9524572649572649,
+      1.6316773504273505,
+      1.0803952991452992,
+      0.8651175213675214,
+      1.4332264957264957,
+      1.3645833333333333,
+      0.8544337606837606,
+      1.2518696581196582,
+      1.1647970085470085,
+      0.40678418803418803,
+      1.6140491452991452,
+      1.2208867521367521,
+      0.28952991452991456,
+      0.7355769230769231,
+      1.2457264957264957,
+      1.3052884615384615,
+      0.9193376068376068,
+      0.4623397435897436,
+      0.5427350427350427,
+      1.422008547008547,
+      1.0750534188034189,
+      0.3803418803418803,
+      0.5592948717948718,
+      0.9388354700854701,
+      1.25,
+      0.7788461538461539,
+      1.2881944444444444,
+      0.9316239316239316,
+      0.5114850427350427,
+      0.9890491452991453,
+      0.4481837606837607,
+      1.5405982905982907,
+      0.5742521367521367,
+      1.0694444444444444,
+      0.3888888888888889,
+      0.20379273504273504,
+      0.46020299145299143,
+      0.8384081196581197,
+      1.0536858974358974,
+      1.4126602564102564,
+      0.594017094017094,
+      0.5833333333333334,
+      0.38274572649572647,
+      0.43856837606837606,
+      0.33226495726495725,
+      0.9417735042735043,
+      1.2013888888888888,
+      1.5587606837606838,
+      1.2799145299145298,
+      0.297275641025641,
+      0.8389423076923077,
+      1.6883012820512822,
+      1.0857371794871795,
+      0.5114850427350427,
+      1.5235042735042734,
+      0.26976495726495725,
+      0.7609508547008547,
+      0.6455662393162394,
+      0.9342948717948718,
+      0.6495726495726496,
+      0.5024038461538461,
+      1.1845619658119657,
+      0.8867521367521367,
+      0.9759615384615384,
+      0.9228098290598291,
+      0.842948717948718,
+      0.6864316239316239,
+      1.3784722222222223,
+      1.4660790598290598,
+      0.4305555555555556,
+      1.2767094017094016,
+      1.5515491452991452,
+      0.7940705128205128,
+      1.5216346153846154,
+      1.4484508547008548,
+      1.2267628205128205,
+      1.392094017094017,
+      1.077991452991453,
+      1.138087606837607,
+      1.2892628205128205,
+      0.688301282051282,
+      1.5678418803418803,
+      0.9481837606837606,
+      1.0320512820512822,
+      1.4588675213675213,
+      0.4724893162393162,
+      0.8002136752136753,
+      1.439102564102564,
+      0.844284188034188,
+      1.0042735042735043,
+      1.265758547008547,
+      1.781784188034188,
+      0.6850961538461539,
+      0.5635683760683761,
+      1.2307692307692308,
+      0.9361645299145299,
+      1.3231837606837606,
+      0.8202457264957265,
+      0.2518696581196581
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.004001918006451888,
+      0.006383970897629227,
+      0.008800339867968012,
+      0.01110061723201374,
+      0.013095963992350917
+    ],
+    "infer_info_gains": [
+      0.9573307045430369,
+      0.5008580546955184,
+      0.3323200289022141,
+      0.23609792172984803,
+      0.1723961393554007
+    ],
+    "infer_delta_probs": [
+      0.0024018655074233043,
+      0.002382052891177339,
+      0.002416368970338786,
+      0.0023002773640457283,
+      0.0019953467603371767
+    ],
+    "infer_expected_reward_accs": [
+      0.40865384615384615,
+      0.9423076923076923,
+      0.9951923076923077,
+      0.9983974358974359,
+      1.0
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      0.9789998306667209,
+      0.8709282117385632,
+      0.8134452134420805,
+      0.7782550232618777,
+      0.7489888666258897
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5506730769230764,
+      0.6017147435897432,
+      0.6277724358974357,
+      0.6431250000000001,
+      0.655929487179488
+    ],
+    "infer_expected_random_set_idxs": [
+      0.8167735042735043,
+      1.0713141025641026,
+      1.5053418803418803,
+      1.0675747863247864,
+      0.8806089743589743,
+      1.2206196581196582,
+      1.1821581196581197,
+      0.719017094017094,
+      1.2628205128205128,
+      1.294871794871795,
+      0.6594551282051282,
+      1.3349358974358974,
+      1.1939102564102564,
+      0.6022970085470085,
+      1.0211004273504274,
+      1.2158119658119657,
+      1.138621794871795,
+      0.9281517094017094,
+      0.49252136752136755,
+      0.6129807692307693,
+      1.1525106837606838,
+      0.9727564102564102,
+      0.4703525641025641,
+      0.7465277777777778,
+      0.9313568376068376,
+      1.0924145299145298,
+      0.8183760683760684,
+      1.231837606837607,
+      0.8883547008547008,
+      0.6599893162393162,
+      1.1538461538461537,
+      0.7326388888888888,
+      1.2673611111111112,
+      0.7516025641025641,
+      0.9628739316239316,
+      0.8287927350427351,
+      0.6677350427350427,
+      0.6068376068376068,
+      0.8693910256410257,
+      1.0082799145299146,
+      1.2195512820512822,
+      0.6284722222222222,
+      0.6477029914529915,
+      0.8167735042735043,
+      0.5809294871794872,
+      0.5886752136752137,
+      0.9903846153846154,
+      1.1621260683760684,
+      1.4997329059829059,
+      1.1965811965811965,
+      0.5841346153846154,
+      0.9719551282051282,
+      1.4679487179487178,
+      1.1452991452991452,
+      0.6543803418803419,
+      1.2099358974358974,
+      0.49919871794871795,
+      0.7596153846153846,
+      0.781517094017094,
+      0.9305555555555556,
+      0.7481303418803419,
+      1.1220619658119657,
+      1.138621794871795,
+      0.9220085470085471,
+      0.9975961538461539,
+      0.8255876068376068,
+      0.905448717948718,
+      0.6335470085470085,
+      1.4035790598290598,
+      1.389155982905983,
+      0.6802884615384616,
+      1.078258547008547,
+      1.4150641025641026,
+      0.9123931623931624,
+      1.2799145299145298,
+      1.2142094017094016,
+      1.1322115384615385,
+      1.279647435897436,
+      0.8303952991452992,
+      1.0096153846153846,
+      1.2099358974358974,
+      0.6375534188034188,
+      1.3683226495726495,
+      1.0197649572649572,
+      1.1503739316239316,
+      1.1599893162393162,
+      0.6084401709401709,
+      0.8563034188034188,
+      1.3712606837606838,
+      0.9073183760683761,
+      0.6754807692307693,
+      1.280715811965812,
+      1.5616987179487178,
+      0.7032585470085471,
+      0.78125,
+      1.1191239316239316,
+      1.0413995726495726,
+      1.1661324786324787,
+      0.7342414529914529,
+      0.5072115384615384
+    ],
+    "kls": [
+      0.0,
+      1.3993024171750827,
+      1.837400524170518,
+      2.0066837602857834,
+      2.0178655405394497,
+      2.036119186654501
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.5126442307692304,
+      0.5515705128205131,
+      0.5793749999999994,
+      0.5925000000000002,
+      0.606137820512821
+    ],
+    "model_random_set_direct_idxs": [
+      1.106837606837607,
+      1.1450320512820513,
+      1.3552350427350428,
+      1.310897435897436,
+      1.0504807692307692,
+      0.6623931623931624,
+      0.7641559829059829,
+      0.7462606837606838,
+      1.0486111111111112,
+      1.4895833333333333,
+      1.2297008547008548,
+      0.8790064102564102,
+      1.329059829059829,
+      0.8974358974358975,
+      1.4001068376068375,
+      1.4121260683760684,
+      1.2222222222222223,
+      0.9652777777777778,
+      0.7286324786324786,
+      0.844551282051282,
+      0.7005876068376068,
+      1.1364850427350428,
+      0.7873931623931624,
+      1.2403846153846154,
+      0.9236111111111112,
+      0.9025106837606838,
+      1.0240384615384615,
+      1.3616452991452992,
+      0.7684294871794872,
+      1.2134081196581197,
+      1.3496260683760684,
+      1.2919337606837606,
+      0.4703525641025641,
+      1.3165064102564104,
+      0.8952991452991453,
+      1.1428952991452992,
+      0.8071581196581197,
+      0.843215811965812,
+      1.0106837606837606,
+      0.7227564102564102,
+      1.2711004273504274,
+      1.2310363247863247,
+      0.7393162393162394,
+      1.1621260683760684,
+      1.0499465811965811,
+      0.6610576923076923,
+      0.8664529914529915,
+      1.3565705128205128,
+      1.2946047008547008,
+      1.1116452991452992,
+      1.1925747863247864,
+      0.842948717948718,
+      1.2278311965811965,
+      1.2684294871794872,
+      1.142094017094017,
+      1.2588141025641026,
+      0.8317307692307693,
+      0.7110042735042735,
+      1.2689636752136753,
+      0.9610042735042735,
+      1.0611645299145298,
+      0.7727029914529915,
+      1.1840277777777777,
+      0.655982905982906,
+      1.2684294871794872,
+      0.9289529914529915,
+      1.015491452991453,
+      0.5328525641025641,
+      0.8464209401709402,
+      1.1754807692307692,
+      1.044871794871795,
+      0.7134081196581197,
+      1.266292735042735,
+      0.8261217948717948,
+      1.2043269230769231,
+      0.7876602564102564,
+      0.8405448717948718,
+      1.0034722222222223,
+      0.8616452991452992,
+      1.2053952991452992,
+      1.1554487179487178,
+      0.7900641025641025,
+      1.0256410256410255,
+      1.0227029914529915,
+      1.2115384615384615,
+      1.0966880341880343,
+      0.6754807692307693,
+      0.7652243589743589,
+      1.111912393162393,
+      1.2606837606837606,
+      0.5245726495726496,
+      1.0934829059829059,
+      1.330662393162393,
+      1.076121794871795,
+      1.3028846153846154,
+      0.7048611111111112,
+      1.030715811965812,
+      0.5021367521367521,
+      0.530715811965812,
+      0.7684294871794872
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7488942307692311,
+      0.7727564102564104,
+      0.7781570512820503,
+      0.7854647435897439,
+      0.7836057692307689
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.7014102564102558,
+      0.7259935897435893,
+      0.7319551282051278,
+      0.7387499999999998,
+      0.7372435897435902
+    ]
+  }
+}

results_v2/flight_uniform_kmin_25_seed0/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_25_seed0.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.6778846153846154,
+      0.7756410256410257,
+      0.8205128205128205,
+      0.8413461538461539,
+      0.8605769230769231
+    ],
+    "model_accs": [
+      0.6778846153846154,
+      0.7756410256410257,
+      0.8205128205128205,
+      0.8413461538461539,
+      0.8605769230769231
+    ],
+    "model_reward_accs": [
+      0.40705128205128205,
+      0.7147435897435898,
+      0.7644230769230769,
+      0.7884615384615384,
+      0.8221153846153846
+    ],
+    "model_consistencies": [
+      0.3733974358974359,
+      0.7596153846153846,
+      0.782051282051282,
+      0.7980769230769231,
+      0.8349358974358975
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.0853461016900794,
+      1.0017586805150902,
+      0.9505566584498683,
+      0.9050267683152515,
+      0.8737652632630708
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.49620192307692373,
+      0.5361057692307691,
+      0.5616506410256411,
+      0.5775801282051284,
+      0.5940224358974369
+    ],
+    "model_random_set_idxs": [
+      0.7102029914529915,
+      1.1845619658119657,
+      1.3028846153846154,
+      1.0868055555555556,
+      1.0093482905982907,
+      1.1169871794871795,
+      1.1241987179487178,
+      0.9332264957264957,
+      1.141559829059829,
+      1.1672008547008548,
+      0.7564102564102564,
+      1.047008547008547,
+      1.21875,
+      0.7743055555555556,
+      1.015224358974359,
+      1.1565170940170941,
+      0.9489850427350427,
+      0.9110576923076923,
+      0.6722756410256411,
+      0.6394230769230769,
+      1.0873397435897436,
+      0.9735576923076923,
+      0.6663995726495726,
+      0.8664529914529915,
+      1.094818376068376,
+      1.0248397435897436,
+      0.9409722222222222,
+      1.2126068376068375,
+      0.7633547008547008,
+      0.9551282051282052,
+      1.109241452991453,
+      0.9898504273504274,
+      0.9457799145299145,
+      0.9177350427350427,
+      0.9572649572649573,
+      0.7860576923076923,
+      0.49412393162393164,
+      0.8362713675213675,
+      1.1036324786324787,
+      1.0291132478632479,
+      1.1581196581196582,
+      0.9561965811965812,
+      0.6888354700854701,
+      0.8341346153846154,
+      0.8541666666666666,
+      0.6426282051282052,
+      0.7711004273504274,
+      1.3239850427350428,
+      1.4738247863247864,
+      1.140224358974359,
+      0.8576388888888888,
+      1.1129807692307692,
+      1.2617521367521367,
+      1.2393162393162394,
+      0.65625,
+      1.2684294871794872,
+      0.6308760683760684,
+      0.782051282051282,
+      0.968215811965812,
+      1.0486111111111112,
+      0.9855769230769231,
+      0.7796474358974359,
+      1.0520833333333333,
+      0.9935897435897436,
+      1.3410790598290598,
+      1.0715811965811965,
+      1.063034188034188,
+      0.6736111111111112,
+      1.1161858974358974,
+      1.1143162393162394,
+      0.7422542735042735,
+      0.9879807692307693,
+      1.2371794871794872,
+      0.8904914529914529,
+      1.2104700854700854,
+      1.0542200854700854,
+      1.0723824786324787,
+      1.1848290598290598,
+      1.045405982905983,
+      1.15625,
+      1.046207264957265,
+      0.8015491452991453,
+      1.2259615384615385,
+      1.1941773504273505,
+      0.9879807692307693,
+      1.1351495726495726,
+      0.7155448717948718,
+      0.7839209401709402,
+      1.1276709401709402,
+      1.018162393162393,
+      0.8621794871794872,
+      1.1482371794871795,
+      1.388621794871795,
+      0.9732905982905983,
+      0.9631410256410257,
+      1.1522435897435896,
+      0.8806089743589743,
+      0.9284188034188035,
+      0.7900641025641025,
+      0.6495726495726496
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.0033278579938500305,
+      0.004670575138036362,
+      0.006049107761535985,
+      0.007047742231324177,
+      0.007102556282497119
+    ],
+    "model_info_gains": [
+      0.03898160032000112,
+      -0.03588045910750368,
+      0.13507502659753112,
+      0.12465967594406419,
+      -0.02581279492931908
+    ],
+    "model_delta_probs": [
+      0.001727805494821447,
+      0.0013427171441863313,
+      0.001378532623499623,
+      0.0009986344697881923,
+      5.481405117294185e-05
+    ],
+    "model_expected_reward_accs": [
+      0.40064102564102566,
+      0.7612179487179487,
+      0.8253205128205128,
+      0.8461538461538461,
+      0.875
+    ],
+    "model_expected_consistencies": [
+      0.375,
+      0.8221153846153846,
+      0.842948717948718,
+      0.8653846153846154,
+      0.8846153846153846
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.063204014042666,
+      0.9674536097209312,
+      0.910174063758459,
+      0.8642988398181877,
+      0.8387069008454231
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5077083333333329,
+      0.5519711538461542,
+      0.5803685897435893,
+      0.6000801282051282,
+      0.6113141025641019
+    ],
+    "model_expected_random_set_idxs": [
+      0.9428418803418803,
+      1.2382478632478633,
+      1.263621794871795,
+      1.0528846153846154,
+      1.0144230769230769,
+      1.0742521367521367,
+      1.2013888888888888,
+      0.8934294871794872,
+      1.234508547008547,
+      1.263621794871795,
+      0.8384081196581197,
+      1.078525641025641,
+      1.1661324786324787,
+      0.8344017094017094,
+      1.0245726495726495,
+      1.2008547008547008,
+      0.9842414529914529,
+      0.8979700854700855,
+      0.6298076923076923,
+      0.6685363247863247,
+      1.1036324786324787,
+      1.048878205128205,
+      0.6412927350427351,
+      0.8661858974358975,
+      1.0908119658119657,
+      0.9839743589743589,
+      0.9254807692307693,
+      1.1931089743589745,
+      0.7767094017094017,
+      0.9714209401709402,
+      1.0929487179487178,
+      0.9722222222222222,
+      0.9847756410256411,
+      0.9722222222222222,
+      0.9305555555555556,
+      0.8715277777777778,
+      0.6014957264957265,
+      0.7863247863247863,
+      1.141292735042735,
+      1.0128205128205128,
+      1.0830662393162394,
+      0.9353632478632479,
+      0.7254273504273504,
+      0.905982905982906,
+      0.8830128205128205,
+      0.6813568376068376,
+      0.7657585470085471,
+      1.2946047008547008,
+      1.4383012820512822,
+      1.077724358974359,
+      0.8712606837606838,
+      1.092681623931624,
+      1.2083333333333333,
+      1.201121794871795,
+      0.6615918803418803,
+      1.2454594017094016,
+      0.6931089743589743,
+      0.7702991452991453,
+      0.9145299145299145,
+      1.0480769230769231,
+      0.9599358974358975,
+      1.1690705128205128,
+      1.0972222222222223,
+      1.0267094017094016,
+      1.2769764957264957,
+      0.9500534188034188,
+      1.0560897435897436,
+      0.5130876068376068,
+      1.0809294871794872,
+      1.1471688034188035,
+      0.7094017094017094,
+      1.0211004273504274,
+      1.2243589743589745,
+      0.9257478632478633,
+      1.1794871794871795,
+      1.0240384615384615,
+      1.0747863247863247,
+      1.248931623931624,
+      0.8541666666666666,
+      1.1153846153846154,
+      1.126068376068376,
+      0.8010149572649573,
+      1.2382478632478633,
+      1.1947115384615385,
+      0.9481837606837606,
+      1.0908119658119657,
+      0.7163461538461539,
+      0.8076923076923077,
+      1.1303418803418803,
+      0.9479166666666666,
+      0.6407585470085471,
+      1.1875,
+      1.3651175213675213,
+      0.9249465811965812,
+      0.9471153846153846,
+      1.0625,
+      0.9196047008547008,
+      0.9700854700854701,
+      0.8619123931623932,
+      0.6372863247863247
+    ],
+    "infer_reward_accs": [
+      0.40384615384615385,
+      0.6073717948717948,
+      0.6666666666666666,
+      0.7708333333333334,
+      0.8365384615384616
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.159648551124148,
+      1.043266400145677,
+      0.951263445491879,
+      0.8997285454197603,
+      0.8428733884675237
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.45897435897435884,
+      0.518333333333333,
+      0.5639262820512819,
+      0.5873237179487184,
+      0.6145352564102569
+    ],
+    "infer_random_set_idxs": [
+      0.4420405982905983,
+      0.9556623931623932,
+      1.6306089743589745,
+      1.1169871794871795,
+      0.9025106837606838,
+      1.4508547008547008,
+      1.3197115384615385,
+      0.8707264957264957,
+      1.2184829059829059,
+      1.1565170940170941,
+      0.4115918803418803,
+      1.6009615384615385,
+      1.2422542735042734,
+      0.32104700854700857,
+      0.7425213675213675,
+      1.2398504273504274,
+      1.3015491452991452,
+      0.9348290598290598,
+      0.4634081196581197,
+      0.5029380341880342,
+      1.3691239316239316,
+      1.0950854700854702,
+      0.3573717948717949,
+      0.5563568376068376,
+      0.8998397435897436,
+      1.2540064102564104,
+      0.7596153846153846,
+      1.2729700854700854,
+      0.9241452991452992,
+      0.4981303418803419,
+      0.9887820512820513,
+      0.44150641025641024,
+      1.5261752136752136,
+      0.5411324786324786,
+      1.0536858974358974,
+      0.40678418803418803,
+      0.23370726495726496,
+      0.4895833333333333,
+      0.8210470085470085,
+      1.0758547008547008,
+      1.3950320512820513,
+      0.6025641025641025,
+      0.5689102564102564,
+      0.39289529914529914,
+      0.4634081196581197,
+      0.3701923076923077,
+      0.9276175213675214,
+      1.1768162393162394,
+      1.5579594017094016,
+      1.250534188034188,
+      0.31330128205128205,
+      0.8282585470085471,
+      1.6498397435897436,
+      1.125,
+      0.5600961538461539,
+      1.4922542735042734,
+      0.2689636752136752,
+      0.780982905982906,
+      0.6490384615384616,
+      0.9254807692307693,
+      0.71875,
+      0.5763888888888888,
+      1.1949786324786325,
+      0.8664529914529915,
+      0.9898504273504274,
+      0.938034188034188,
+      0.8360042735042735,
+      0.6984508547008547,
+      1.3218482905982907,
+      1.4083867521367521,
+      0.45699786324786323,
+      1.2606837606837606,
+      1.5734508547008548,
+      0.8453525641025641,
+      1.5389957264957266,
+      1.3693910256410255,
+      1.1826923076923077,
+      1.3787393162393162,
+      1.0373931623931625,
+      1.108707264957265,
+      1.3084935897435896,
+      0.6781517094017094,
+      1.549412393162393,
+      0.9385683760683761,
+      1.0229700854700854,
+      1.4487179487179487,
+      0.5072115384615384,
+      0.8175747863247863,
+      1.4415064102564104,
+      0.8565705128205128,
+      0.96875,
+      1.2735042735042734,
+      1.780181623931624,
+      0.7208867521367521,
+      0.5841346153846154,
+      1.2379807692307692,
+      0.9292200854700855,
+      1.3338675213675213,
+      0.8151709401709402,
+      0.27564102564102566
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.004118538420371829,
+      0.006831882939131435,
+      0.00984694722033871,
+      0.012855215985965538,
+      0.015662717184102056
+    ],
+    "infer_info_gains": [
+      0.9802047100631385,
+      0.5446994138538404,
+      0.389384612231674,
+      0.2761803102866448,
+      0.2008648030039475
+    ],
+    "infer_delta_probs": [
+      0.0025184859213432456,
+      0.002713344518759606,
+      0.003015064281207276,
+      0.0030082687656268273,
+      0.0028075011981365185
+    ],
+    "infer_expected_reward_accs": [
+      0.40064102564102566,
+      0.9022435897435898,
+      0.9903846153846154,
+      0.9983974358974359,
+      1.0
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      0.9890784901554306,
+      0.8592106665470579,
+      0.791047944148404,
+      0.7456433471913758,
+      0.7188631261697093
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5422916666666664,
+      0.6043589743589748,
+      0.6375961538461533,
+      0.6597756410256408,
+      0.6705448717948713
+    ],
+    "infer_expected_random_set_idxs": [
+      0.875,
+      1.0507478632478633,
+      1.4949252136752136,
+      1.0916132478632479,
+      0.9257478632478633,
+      1.2502670940170941,
+      1.1650641025641026,
+      0.7652243589743589,
+      1.2540064102564104,
+      1.2849893162393162,
+      0.6447649572649573,
+      1.3330662393162394,
+      1.2238247863247864,
+      0.6156517094017094,
+      0.9962606837606838,
+      1.2003205128205128,
+      1.1589209401709402,
+      0.907051282051282,
+      0.46955128205128205,
+      0.562767094017094,
+      1.125,
+      0.9866452991452992,
+      0.4358974358974359,
+      0.7008547008547008,
+      0.8926282051282052,
+      1.080662393162393,
+      0.7828525641025641,
+      1.2094017094017093,
+      0.8952991452991453,
+      0.6522435897435898,
+      1.1444978632478633,
+      0.7318376068376068,
+      1.2529380341880343,
+      0.7422542735042735,
+      0.9618055555555556,
+      0.8629807692307693,
+      0.6474358974358975,
+      0.65625,
+      0.8664529914529915,
+      1.0683760683760684,
+      1.1797542735042734,
+      0.6199252136752137,
+      0.6153846153846154,
+      0.8189102564102564,
+      0.5726495726495726,
+      0.6076388888888888,
+      1.0050747863247864,
+      1.1615918803418803,
+      1.5048076923076923,
+      1.1816239316239316,
+      0.5675747863247863,
+      0.9305555555555556,
+      1.4780982905982907,
+      1.1594551282051282,
+      0.7080662393162394,
+      1.1981837606837606,
+      0.48023504273504275,
+      0.750267094017094,
+      0.7908653846153846,
+      0.9166666666666666,
+      0.8074252136752137,
+      1.1963141025641026,
+      1.1463675213675213,
+      0.9126602564102564,
+      1.014957264957265,
+      0.8346688034188035,
+      0.9081196581196581,
+      0.6503739316239316,
+      1.3675213675213675,
+      1.327724358974359,
+      0.6864316239316239,
+      1.0649038461538463,
+      1.4230769230769231,
+      0.9145299145299145,
+      1.342147435897436,
+      1.1597222222222223,
+      1.0964209401709402,
+      1.2793803418803418,
+      0.7922008547008547,
+      0.9772970085470085,
+      1.2572115384615385,
+      0.6431623931623932,
+      1.3808760683760684,
+      1.0189636752136753,
+      1.1201923076923077,
+      1.171207264957265,
+      0.5956196581196581,
+      0.8629807692307693,
+      1.3501602564102564,
+      0.8808760683760684,
+      0.6495726495726496,
+      1.2820512820512822,
+      1.5723824786324787,
+      0.6933760683760684,
+      0.7836538461538461,
+      1.1455662393162394,
+      1.0251068376068375,
+      1.1677350427350428,
+      0.7203525641025641,
+      0.5267094017094017
+    ],
+    "kls": [
+      0.0,
+      1.4385260392291757,
+      1.868731081808458,
+      2.065342381674143,
+      2.1286679936291413,
+      2.1813410252906538
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.507195512820513,
+      0.5455128205128211,
+      0.5697275641025639,
+      0.5871314102564106,
+      0.6006089743589742
+    ],
+    "model_random_set_direct_idxs": [
+      1.1623931623931625,
+      1.1388888888888888,
+      1.4147970085470085,
+      1.3269230769230769,
+      1.1030982905982907,
+      0.6645299145299145,
+      0.7152777777777778,
+      0.7737713675213675,
+      1.0547542735042734,
+      1.516025641025641,
+      1.1901709401709402,
+      0.8584401709401709,
+      1.3672542735042734,
+      0.8993055555555556,
+      1.3717948717948718,
+      1.3787393162393162,
+      1.234241452991453,
+      0.9577991452991453,
+      0.6853632478632479,
+      0.8474893162393162,
+      0.6618589743589743,
+      1.1931089743589745,
+      0.780715811965812,
+      1.205128205128205,
+      0.8864850427350427,
+      0.8552350427350427,
+      1.0077457264957266,
+      1.3175747863247864,
+      0.7972756410256411,
+      1.202991452991453,
+      1.3293269230769231,
+      1.297008547008547,
+      0.4751602564102564,
+      1.2719017094017093,
+      0.8731303418803419,
+      1.1340811965811965,
+      0.8277243589743589,
+      0.8480235042735043,
+      0.9893162393162394,
+      0.7516025641025641,
+      1.2508012820512822,
+      1.2115384615384615,
+      0.7107371794871795,
+      1.1511752136752136,
+      1.0352564102564104,
+      0.6551816239316239,
+      0.8528311965811965,
+      1.3245192307692308,
+      1.328258547008547,
+      1.0910790598290598,
+      1.1931089743589745,
+      0.7823183760683761,
+      1.264690170940171,
+      1.2767094017094016,
+      1.173878205128205,
+      1.2075320512820513,
+      0.8207799145299145,
+      0.7286324786324786,
+      1.3410790598290598,
+      0.9706196581196581,
+      1.0878739316239316,
+      0.8501602564102564,
+      1.1653311965811965,
+      0.6541132478632479,
+      1.2524038461538463,
+      0.9313568376068376,
+      0.9831730769230769,
+      0.5333867521367521,
+      0.8424145299145299,
+      1.1631944444444444,
+      1.045940170940171,
+      0.6714743589743589,
+      1.2876602564102564,
+      0.8466880341880342,
+      1.2566773504273505,
+      0.7524038461538461,
+      0.8223824786324786,
+      1.0438034188034189,
+      0.8189102564102564,
+      1.1482371794871795,
+      1.186431623931624,
+      0.7916666666666666,
+      1.0293803418803418,
+      0.9834401709401709,
+      1.2112713675213675,
+      1.0982905982905984,
+      0.6690705128205128,
+      0.812232905982906,
+      1.0910790598290598,
+      1.2556089743589745,
+      0.4890491452991453,
+      1.1060363247863247,
+      1.3480235042735043,
+      1.0512820512820513,
+      1.3194444444444444,
+      0.7299679487179487,
+      0.9933226495726496,
+      0.4877136752136752,
+      0.5069444444444444,
+      0.7580128205128205
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7465865384615376,
+      0.7665544871794867,
+      0.7694871794871789,
+      0.7717147435897429,
+      0.778798076923076
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.6962179487179486,
+      0.7164102564102562,
+      0.7206891025641016,
+      0.7227724358974361,
+      0.7334134615384622
+    ]
+  }
+}

results_v2/flight_uniform_kmin_2_seed1/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_2_seed1.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.8573717948717948,
+      0.9182692307692307,
+      0.9407051282051282,
+      0.9423076923076923,
+      0.9535256410256411
+    ],
+    "model_accs": [
+      0.8573717948717948,
+      0.9182692307692307,
+      0.9407051282051282,
+      0.9423076923076923,
+      0.9535256410256411
+    ],
+    "model_reward_accs": [
+      0.4599358974358974,
+      0.9086538461538461,
+      0.9551282051282052,
+      0.9583333333333334,
+      0.9759615384615384
+    ],
+    "model_consistencies": [
+      0.4567307692307692,
+      0.9182692307692307,
+      0.9471153846153846,
+      0.9631410256410257,
+      0.9599358974358975
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.1089842505249283,
+      1.0546895720929566,
+      1.0217170014123984,
+      1.001026128900347,
+      1.0060654848271091
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.48440705128205125,
+      0.5112019230769226,
+      0.5275160256410264,
+      0.5371153846153849,
+      0.5331089743589746
+    ],
+    "model_random_set_idxs": [
+      0.7422542735042735,
+      1.1861645299145298,
+      1.3138354700854702,
+      1.0830662393162394,
+      1.0194978632478633,
+      1.1183226495726495,
+      1.111912393162393,
+      0.9228098290598291,
+      1.1038995726495726,
+      1.1450320512820513,
+      0.7678952991452992,
+      1.077991452991453,
+      1.2019230769230769,
+      0.8282585470085471,
+      1.048878205128205,
+      1.1375534188034189,
+      0.9011752136752137,
+      0.9583333333333334,
+      0.6915064102564102,
+      0.6335470085470085,
+      1.0523504273504274,
+      0.9599358974358975,
+      0.6621260683760684,
+      0.8766025641025641,
+      1.0745192307692308,
+      1.0603632478632479,
+      0.9166666666666666,
+      1.1583867521367521,
+      0.7713675213675214,
+      0.9150641025641025,
+      1.1004273504273505,
+      0.9810363247863247,
+      0.9364316239316239,
+      0.875,
+      0.9917200854700855,
+      0.750267094017094,
+      0.5408653846153846,
+      0.7983440170940171,
+      1.0982905982905984,
+      0.9644764957264957,
+      1.1180555555555556,
+      0.9842414529914529,
+      0.6463675213675214,
+      0.8084935897435898,
+      0.8571047008547008,
+      0.657051282051282,
+      0.7724358974358975,
+      1.297008547008547,
+      1.4898504273504274,
+      1.063034188034188,
+      0.8448183760683761,
+      1.093215811965812,
+      1.2438568376068375,
+      1.2542735042735043,
+      0.6744123931623932,
+      1.2278311965811965,
+      0.6143162393162394,
+      0.7871260683760684,
+      0.9943910256410257,
+      1.109508547008547,
+      0.9588675213675214,
+      0.7668269230769231,
+      1.0389957264957266,
+      0.9740918803418803,
+      1.3429487179487178,
+      1.061431623931624,
+      1.048878205128205,
+      0.7110042735042735,
+      1.109241452991453,
+      1.1434294871794872,
+      0.750267094017094,
+      0.9671474358974359,
+      1.2596153846153846,
+      0.9182692307692307,
+      1.21875,
+      1.0742521367521367,
+      1.0854700854700854,
+      1.172008547008547,
+      1.014957264957265,
+      1.1471688034188035,
+      1.060897435897436,
+      0.8028846153846154,
+      1.203525641025641,
+      1.232905982905983,
+      1.0138888888888888,
+      1.1514423076923077,
+      0.6794871794871795,
+      0.7935363247863247,
+      1.1690705128205128,
+      1.0261752136752136,
+      0.8322649572649573,
+      1.1618589743589745,
+      1.3605769230769231,
+      0.9567307692307693,
+      0.9658119658119658,
+      1.0723824786324787,
+      0.9364316239316239,
+      0.938301282051282,
+      0.7660256410256411,
+      0.6412927350427351
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.002899432562330162,
+      0.003163518561349449,
+      0.0034104676652689024,
+      0.003947907363480199,
+      0.003918906757233865
+    ],
+    "model_info_gains": [
+      -0.29783388060716964,
+      -0.3180377852829086,
+      -0.08764133495053394,
+      -0.04463135386968824,
+      -0.05661404596469076
+    ],
+    "model_delta_probs": [
+      0.0012993800633015785,
+      0.0002640859990192869,
+      0.0002469491039194534,
+      0.0005374396982112964,
+      -2.9000606246333584e-05
+    ],
+    "model_expected_reward_accs": [
+      0.4599358974358974,
+      0.9246794871794872,
+      0.9583333333333334,
+      0.9615384615384616,
+      0.9759615384615384
+    ],
+    "model_expected_consistencies": [
+      0.46314102564102566,
+      0.9342948717948718,
+      0.9631410256410257,
+      0.9727564102564102,
+      0.9663461538461539
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.0813388639644954,
+      1.0397892672592857,
+      1.0049248128607255,
+      0.9865549159247495,
+      0.9831149917943737
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.49796474358974363,
+      0.5182371794871796,
+      0.5346314102564101,
+      0.5443269230769227,
+      0.5446634615384613
+    ],
+    "model_expected_random_set_idxs": [
+      0.9615384615384616,
+      1.2280982905982907,
+      1.2913995726495726,
+      1.0347222222222223,
+      1.0269764957264957,
+      1.0956196581196582,
+      1.1768162393162394,
+      0.8659188034188035,
+      1.2238247863247864,
+      1.233440170940171,
+      0.8255876068376068,
+      1.1076388888888888,
+      1.1832264957264957,
+      0.8667200854700855,
+      1.0683760683760684,
+      1.1925747863247864,
+      0.9300213675213675,
+      0.9460470085470085,
+      0.6506410256410257,
+      0.6519764957264957,
+      1.0747863247863247,
+      1.0253739316239316,
+      0.6161858974358975,
+      0.8739316239316239,
+      1.0713141025641026,
+      1.0069444444444444,
+      0.9102564102564102,
+      1.1650641025641026,
+      0.812767094017094,
+      0.9286858974358975,
+      1.080128205128205,
+      0.9602029914529915,
+      0.9703525641025641,
+      0.9444444444444444,
+      0.9569978632478633,
+      0.8189102564102564,
+      0.6431623931623932,
+      0.7566773504273504,
+      1.1485042735042734,
+      0.9540598290598291,
+      1.030715811965812,
+      0.9599358974358975,
+      0.6704059829059829,
+      0.8736645299145299,
+      0.8819444444444444,
+      0.6786858974358975,
+      0.7486645299145299,
+      1.2751068376068375,
+      1.4620726495726495,
+      1.0304487179487178,
+      0.8600427350427351,
+      1.093215811965812,
+      1.2048611111111112,
+      1.2227564102564104,
+      0.6760149572649573,
+      1.2131410256410255,
+      0.6650641025641025,
+      0.7678952991452992,
+      0.9423076923076923,
+      1.106837606837607,
+      0.9519230769230769,
+      1.169871794871795,
+      1.0699786324786325,
+      1.0106837606837606,
+      1.3095619658119657,
+      0.9356303418803419,
+      1.0405982905982907,
+      0.5373931623931624,
+      1.0830662393162394,
+      1.1701388888888888,
+      0.7232905982905983,
+      1.0024038461538463,
+      1.2473290598290598,
+      0.9433760683760684,
+      1.200587606837607,
+      1.0419337606837606,
+      1.1017628205128205,
+      1.2435897435897436,
+      0.8450854700854701,
+      1.1076388888888888,
+      1.108440170940171,
+      0.7868589743589743,
+      1.2211538461538463,
+      1.2302350427350428,
+      0.9791666666666666,
+      1.110309829059829,
+      0.6744123931623932,
+      0.8055555555555556,
+      1.1594551282051282,
+      0.9220085470085471,
+      0.6292735042735043,
+      1.2099358974358974,
+      1.3394764957264957,
+      0.9118589743589743,
+      0.9551282051282052,
+      1.0069444444444444,
+      0.9580662393162394,
+      0.9869123931623932,
+      0.8509615384615384,
+      0.6463675213675214
+    ],
+    "infer_reward_accs": [
+      0.4599358974358974,
+      0.7483974358974359,
+      0.8653846153846154,
+      0.8990384615384616,
+      0.9423076923076923
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.2041652943467356,
+      1.1512602201203603,
+      1.1312578130613953,
+      1.1082860383781192,
+      1.0999489497190913
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.4363782051282046,
+      0.46671474358974424,
+      0.47852564102564144,
+      0.48871794871794866,
+      0.4920993589743597
+    ],
+    "infer_random_set_idxs": [
+      0.30074786324786323,
+      0.9404380341880342,
+      1.7224893162393162,
+      0.9962606837606838,
+      0.8878205128205128,
+      1.5283119658119657,
+      1.4455128205128205,
+      0.8461538461538461,
+      1.3418803418803418,
+      1.1383547008547008,
+      0.29246794871794873,
+      1.7550747863247864,
+      1.1965811965811965,
+      0.23397435897435898,
+      0.6645299145299145,
+      1.236912393162393,
+      1.2719017094017093,
+      0.9006410256410257,
+      0.39823717948717946,
+      0.4791666666666667,
+      1.5256410256410255,
+      1.0579594017094016,
+      0.29246794871794873,
+      0.5208333333333334,
+      0.9425747863247863,
+      1.2780448717948718,
+      0.6597222222222222,
+      1.2759081196581197,
+      0.9783653846153846,
+      0.39636752136752135,
+      0.9337606837606838,
+      0.25587606837606836,
+      1.6525106837606838,
+      0.42574786324786323,
+      1.1284722222222223,
+      0.2668269230769231,
+      0.10924145299145299,
+      0.33653846153846156,
+      0.8560363247863247,
+      1.0261752136752136,
+      1.4503205128205128,
+      0.4519230769230769,
+      0.4858440170940171,
+      0.25053418803418803,
+      0.3643162393162393,
+      0.1920405982905983,
+      0.9316239316239316,
+      1.1808226495726495,
+      1.6351495726495726,
+      1.25,
+      0.16452991452991453,
+      0.8186431623931624,
+      1.7897970085470085,
+      1.0400641025641026,
+      0.39823717948717946,
+      1.6177884615384615,
+      0.14610042735042736,
+      0.7745726495726496,
+      0.592948717948718,
+      0.9754273504273504,
+      0.5405982905982906,
+      0.46474358974358976,
+      1.1658653846153846,
+      0.9284188034188035,
+      0.9754273504273504,
+      0.8822115384615384,
+      0.8584401709401709,
+      0.6995192307692307,
+      1.5248397435897436,
+      1.625,
+      0.24786324786324787,
+      1.3704594017094016,
+      1.6615918803418803,
+      0.8151709401709402,
+      1.5694444444444444,
+      1.5232371794871795,
+      1.3496260683760684,
+      1.5128205128205128,
+      1.0304487179487178,
+      1.1290064102564104,
+      1.3875534188034189,
+      0.6145833333333334,
+      1.6605235042735043,
+      0.9751602564102564,
+      1.0037393162393162,
+      1.5520833333333333,
+      0.29246794871794873,
+      0.8143696581196581,
+      1.5365918803418803,
+      0.7518696581196581,
+      1.032318376068376,
+      1.3832799145299146,
+      1.891292735042735,
+      0.5566239316239316,
+      0.422275641025641,
+      1.1939102564102564,
+      0.9751602564102564,
+      1.4652777777777777,
+      0.8274572649572649,
+      0.20005341880341881
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.0035496750275540714,
+      0.004596184088955059,
+      0.005332796426040285,
+      0.005873450517822512,
+      0.006262582686892925
+    ],
+    "infer_info_gains": [
+      0.8376224052745943,
+      0.2708003673794884,
+      0.1498634056696228,
+      0.09949425793449776,
+      0.0699434656343728
+    ],
+    "infer_delta_probs": [
+      0.0019496225285254882,
+      0.001046509061400987,
+      0.000736612337085227,
+      0.0005406540917822265,
+      0.00038913216907041293
+    ],
+    "infer_expected_reward_accs": [
+      0.4599358974358974,
+      1.0,
+      1.0,
+      1.0,
+      1.0
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      1.0335349429699907,
+      0.991976966475302,
+      0.968086348275788,
+      0.9512620047977273,
+      0.9442735070165102
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5254326923076931,
+      0.5471153846153849,
+      0.5575641025641033,
+      0.5658012820512829,
+      0.5690224358974361
+    ],
+    "infer_expected_random_set_idxs": [
+      0.8034188034188035,
+      1.0956196581196582,
+      1.561431623931624,
+      1.0496794871794872,
+      0.8635149572649573,
+      1.3028846153846154,
+      1.1549145299145298,
+      0.7318376068376068,
+      1.2887286324786325,
+      1.2868589743589745,
+      0.6164529914529915,
+      1.4270833333333333,
+      1.2305021367521367,
+      0.6151175213675214,
+      1.0333867521367521,
+      1.2321047008547008,
+      1.1599893162393162,
+      0.9105235042735043,
+      0.4495192307692308,
+      0.5876068376068376,
+      1.109241452991453,
+      0.9545940170940171,
+      0.42441239316239315,
+      0.7131410256410257,
+      0.8862179487179487,
+      1.1298076923076923,
+      0.7588141025641025,
+      1.2403846153846154,
+      0.8677884615384616,
+      0.5683760683760684,
+      1.1818910256410255,
+      0.6848290598290598,
+      1.3677884615384615,
+      0.7043269230769231,
+      1.0024038461538463,
+      0.8501602564102564,
+      0.6346153846153846,
+      0.5793269230769231,
+      0.7780448717948718,
+      0.9954594017094017,
+      1.202724358974359,
+      0.5515491452991453,
+      0.6335470085470085,
+      0.7596153846153846,
+      0.5186965811965812,
+      0.5085470085470085,
+      1.0769230769230769,
+      1.1423611111111112,
+      1.5074786324786325,
+      1.1797542735042734,
+      0.47622863247863245,
+      0.9324252136752137,
+      1.53125,
+      1.1137820512820513,
+      0.6952457264957265,
+      1.1634615384615385,
+      0.44150641025641024,
+      0.7558760683760684,
+      0.7895299145299145,
+      0.9284188034188035,
+      0.6875,
+      1.1842948717948718,
+      1.156784188034188,
+      0.9468482905982906,
+      0.999732905982906,
+      0.7927350427350427,
+      0.9091880341880342,
+      0.6997863247863247,
+      1.5122863247863247,
+      1.4660790598290598,
+      0.6282051282051282,
+      1.0603632478632479,
+      1.4569978632478633,
+      0.8688568376068376,
+      1.327457264957265,
+      1.1674679487179487,
+      1.1057692307692308,
+      1.3803418803418803,
+      0.7881944444444444,
+      0.9433760683760684,
+      1.2497329059829059,
+      0.6127136752136753,
+      1.4420405982905984,
+      1.001602564102564,
+      1.189102564102564,
+      1.1947115384615385,
+      0.5288461538461539,
+      0.8624465811965812,
+      1.4399038461538463,
+      0.8856837606837606,
+      0.6471688034188035,
+      1.3458867521367521,
+      1.6284722222222223,
+      0.6450320512820513,
+      0.7347756410256411,
+      1.1324786324786325,
+      1.0427350427350428,
+      1.2230235042735043,
+      0.6848290598290598,
+      0.453525641025641
+    ],
+    "kls": [
+      0.0,
+      1.4634322799215775,
+      1.881804249943822,
+      2.026648560254072,
+      2.0813687295727417,
+      2.064514135994517
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.5053525641025641,
+      0.5259775641025641,
+      0.5367147435897439,
+      0.5478846153846154,
+      0.5537660256410261
+    ],
+    "model_random_set_direct_idxs": [
+      1.172275641025641,
+      1.1589209401709402,
+      1.3883547008547008,
+      1.279647435897436,
+      1.080128205128205,
+      0.6845619658119658,
+      0.719017094017094,
+      0.7759081196581197,
+      1.0964209401709402,
+      1.516292735042735,
+      1.201655982905983,
+      0.8827457264957265,
+      1.3469551282051282,
+      0.9313568376068376,
+      1.3838141025641026,
+      1.404647435897436,
+      1.2075320512820513,
+      0.9783653846153846,
+      0.7053952991452992,
+      0.8416132478632479,
+      0.6530448717948718,
+      1.1533119658119657,
+      0.7665598290598291,
+      1.2366452991452992,
+      0.9126602564102564,
+      0.9289529914529915,
+      0.9965277777777778,
+      1.3410790598290598,
+      0.8199786324786325,
+      1.1645299145299146,
+      1.358974358974359,
+      1.3106303418803418,
+      0.4636752136752137,
+      1.2849893162393162,
+      0.9190705128205128,
+      1.1356837606837606,
+      0.8787393162393162,
+      0.8512286324786325,
+      0.9933226495726496,
+      0.7345085470085471,
+      1.2462606837606838,
+      1.217681623931624,
+      0.6965811965811965,
+      1.1474358974358974,
+      1.061965811965812,
+      0.6741452991452992,
+      0.8811431623931624,
+      1.3357371794871795,
+      1.329059829059829,
+      1.0982905982905984,
+      1.1981837606837606,
+      0.8191773504273504,
+      1.2617521367521367,
+      1.2820512820512822,
+      1.2144764957264957,
+      1.1936431623931625,
+      0.78125,
+      0.7200854700854701,
+      1.3704594017094016,
+      1.0045405982905984,
+      1.0966880341880343,
+      0.8464209401709402,
+      1.1800213675213675,
+      0.6637286324786325,
+      1.3223824786324787,
+      0.9188034188034188,
+      1.0355235042735043,
+      0.5758547008547008,
+      0.8613782051282052,
+      1.1976495726495726,
+      1.076121794871795,
+      0.6704059829059829,
+      1.280715811965812,
+      0.8405448717948718,
+      1.2232905982905984,
+      0.7470619658119658,
+      0.8373397435897436,
+      1.0267094017094016,
+      0.8528311965811965,
+      1.1439636752136753,
+      1.1589209401709402,
+      0.7681623931623932,
+      1.0344551282051282,
+      1.0349893162393162,
+      1.220352564102564,
+      1.0868055555555556,
+      0.6311431623931624,
+      0.8063568376068376,
+      1.1367521367521367,
+      1.2727029914529915,
+      0.5061431623931624,
+      1.1124465811965811,
+      1.3229166666666667,
+      1.0331196581196582,
+      1.3416132478632479,
+      0.6899038461538461,
+      1.0600961538461537,
+      0.49145299145299143,
+      0.5034722222222222,
+      0.7913995726495726
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.748108974358974,
+      0.7745833333333332,
+      0.7807051282051279,
+      0.7812019230769225,
+      0.7846634615384612
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.7030448717948714,
+      0.726842948717949,
+      0.7343749999999992,
+      0.740576923076923,
+      0.7390544871794871
+    ]
+  }
+}

results_v2/flight_uniform_kmin_36_seed1/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_36_seed1.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.5641025641025641,
+      0.6602564102564102,
+      0.7163461538461539,
+      0.7115384615384616,
+      0.7628205128205128
+    ],
+    "model_accs": [
+      0.5641025641025641,
+      0.6602564102564102,
+      0.7163461538461539,
+      0.7115384615384616,
+      0.7628205128205128
+    ],
+    "model_reward_accs": [
+      0.40384615384615385,
+      0.5993589743589743,
+      0.7051282051282052,
+      0.7259615384615384,
+      0.7467948717948718
+    ],
+    "model_consistencies": [
+      0.34455128205128205,
+      0.6522435897435898,
+      0.7371794871794872,
+      0.7307692307692307,
+      0.7740384615384616
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.1013583814937558,
+      0.9548699135020138,
+      0.9083161853594474,
+      0.8868733253086925,
+      0.8475515342322106
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.4828685897435901,
+      0.5555608974358978,
+      0.5758173076923077,
+      0.5873076923076924,
+      0.603958333333333
+    ],
+    "model_random_set_idxs": [
+      0.7248931623931624,
+      1.2112713675213675,
+      1.2767094017094016,
+      1.077457264957265,
+      1.0144230769230769,
+      1.0961538461538463,
+      1.1506410256410255,
+      0.9452457264957265,
+      1.1279380341880343,
+      1.1375534188034189,
+      0.7852564102564102,
+      1.0590277777777777,
+      1.2096688034188035,
+      0.8157051282051282,
+      1.0408653846153846,
+      1.1653311965811965,
+      0.9137286324786325,
+      0.9097222222222222,
+      0.7094017094017094,
+      0.6680021367521367,
+      1.0731837606837606,
+      0.9818376068376068,
+      0.7024572649572649,
+      0.8944978632478633,
+      1.1356837606837606,
+      1.0275106837606838,
+      0.9487179487179487,
+      1.189102564102564,
+      0.7537393162393162,
+      0.9618055555555556,
+      1.0990918803418803,
+      1.0320512820512822,
+      0.9190705128205128,
+      0.9505876068376068,
+      0.9569978632478633,
+      0.7978098290598291,
+      0.5114850427350427,
+      0.8544337606837606,
+      1.1241987179487178,
+      0.9967948717948718,
+      1.1148504273504274,
+      0.9797008547008547,
+      0.6802884615384616,
+      0.8482905982905983,
+      0.8832799145299145,
+      0.65625,
+      0.7524038461538461,
+      1.3376068376068375,
+      1.438034188034188,
+      1.1105769230769231,
+      0.8725961538461539,
+      1.1506410256410255,
+      1.2131410256410255,
+      1.2417200854700854,
+      0.6650641025641025,
+      1.2769764957264957,
+      0.6450320512820513,
+      0.7772435897435898,
+      0.9545940170940171,
+      1.0600961538461537,
+      0.9970619658119658,
+      0.7465277777777778,
+      1.0248397435897436,
+      1.0024038461538463,
+      1.3731303418803418,
+      1.0657051282051282,
+      1.0964209401709402,
+      0.6805555555555556,
+      1.0830662393162394,
+      1.0876068376068375,
+      0.7654914529914529,
+      1.0120192307692308,
+      1.200587606837607,
+      0.8739316239316239,
+      1.1789529914529915,
+      1.0520833333333333,
+      1.0446047008547008,
+      1.1909722222222223,
+      1.0638354700854702,
+      1.1989850427350428,
+      1.0320512820512822,
+      0.8141025641025641,
+      1.233707264957265,
+      1.2077991452991452,
+      0.9658119658119658,
+      1.140491452991453,
+      0.718215811965812,
+      0.7465277777777778,
+      1.1461004273504274,
+      1.0213675213675213,
+      0.8635149572649573,
+      1.140758547008547,
+      1.375534188034188,
+      1.0080128205128205,
+      0.9858440170940171,
+      1.1228632478632479,
+      0.8611111111111112,
+      0.905448717948718,
+      0.7964743589743589,
+      0.6335470085470085
+    ],
+    "model_probs": [
+      0.0016002127930928277,
+      0.003936984390688572,
+      0.005429787747427912,
+      0.006445559058708483,
+      0.007796346421475266,
+      0.007733882042703208
+    ],
+    "model_info_gains": [
+      0.14786468785171497,
+      0.11344116601701973,
+      0.05599089674889461,
+      0.012977222274893526,
+      0.1422713255939876
+    ],
+    "model_delta_probs": [
+      0.0023367715975957447,
+      0.0014928033567393401,
+      0.0010157713112805711,
+      0.0013507873627667825,
+      -6.246437877205784e-05
+    ],
+    "model_expected_reward_accs": [
+      0.40384615384615385,
+      0.655448717948718,
+      0.7403846153846154,
+      0.7692307692307693,
+      0.7868589743589743
+    ],
+    "model_expected_consistencies": [
+      0.34455128205128205,
+      0.7275641025641025,
+      0.8157051282051282,
+      0.7868589743589743,
+      0.8125
+    ],
+    "model_expected_l2s": [
+      1.356186067872485,
+      1.0628945508962229,
+      0.9376255270530013,
+      0.8780245415766341,
+      0.8485038611897119,
+      0.8028431715237172
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5024358974358972,
+      0.5628044871794872,
+      0.5929487179487178,
+      0.606826923076923,
+      0.6253044871794863
+    ],
+    "model_expected_random_set_idxs": [
+      0.9647435897435898,
+      1.2534722222222223,
+      1.235309829059829,
+      1.0331196581196582,
+      1.0050747863247864,
+      1.0769230769230769,
+      1.2259615384615385,
+      0.90625,
+      1.2433226495726495,
+      1.2566773504273505,
+      0.8776709401709402,
+      1.0699786324786325,
+      1.170405982905983,
+      0.8563034188034188,
+      1.0357905982905984,
+      1.2128739316239316,
+      0.9489850427350427,
+      0.8931623931623932,
+      0.6802884615384616,
+      0.6971153846153846,
+      1.1167200854700854,
+      1.0531517094017093,
+      0.6837606837606838,
+      0.8990384615384616,
+      1.1298076923076923,
+      0.9727564102564102,
+      0.9447115384615384,
+      1.1925747863247864,
+      0.7761752136752137,
+      0.9826388888888888,
+      1.0705128205128205,
+      1.0085470085470085,
+      0.9604700854700855,
+      1.0237713675213675,
+      0.9294871794871795,
+      0.8846153846153846,
+      0.6108440170940171,
+      0.8173076923076923,
+      1.1469017094017093,
+      0.9874465811965812,
+      1.0502136752136753,
+      0.9743589743589743,
+      0.7251602564102564,
+      0.9305555555555556,
+      0.9172008547008547,
+      0.6920405982905983,
+      0.7556089743589743,
+      1.3237179487179487,
+      1.4027777777777777,
+      1.0566239316239316,
+      0.8993055555555556,
+      1.142094017094017,
+      1.1511752136752136,
+      1.1949786324786325,
+      0.6594551282051282,
+      1.251602564102564,
+      0.7086004273504274,
+      0.750267094017094,
+      0.9129273504273504,
+      1.078525641025641,
+      0.9778311965811965,
+      1.1554487179487178,
+      1.0884081196581197,
+      1.0363247863247864,
+      1.3173076923076923,
+      0.9449786324786325,
+      1.0884081196581197,
+      0.500801282051282,
+      1.0697115384615385,
+      1.1209935897435896,
+      0.7355769230769231,
+      1.0576923076923077,
+      1.1808226495726495,
+      0.9003739316239316,
+      1.1565170940170941,
+      1.0243055555555556,
+      1.0566239316239316,
+      1.2598824786324787,
+      0.8822115384615384,
+      1.1514423076923077,
+      1.0886752136752136,
+      0.8170405982905983,
+      1.2633547008547008,
+      1.2003205128205128,
+      0.9305555555555556,
+      1.0822649572649572,
+      0.7238247863247863,
+      0.7719017094017094,
+      1.1495726495726495,
+      0.9265491452991453,
+      0.6442307692307693,
+      1.1800213675213675,
+      1.3397435897435896,
+      0.9740918803418803,
+      0.9719551282051282,
+      1.0245726495726495,
+      0.9089209401709402,
+      0.9439102564102564,
+      0.8776709401709402,
+      0.6402243589743589
+    ],
+    "infer_reward_accs": [
+      0.40384615384615385,
+      0.5272435897435898,
+      0.6330128205128205,
+      0.7211538461538461,
+      0.7932692307692307
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.164475425932949,
+      1.005596483776882,
+      0.9004039148924082,
+      0.8077634821907682,
+      0.7422787683466192
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.45027243589743565,
+      0.5360256410256412,
+      0.5851602564102564,
+      0.6272435897435897,
+      0.6588141025641023
+    ],
+    "infer_random_set_idxs": [
+      0.5064102564102564,
+      0.9524572649572649,
+      1.6279380341880343,
+      1.0902777777777777,
+      0.9489850427350427,
+      1.4118589743589745,
+      1.2724358974358974,
+      0.8603098290598291,
+      1.2596153846153846,
+      1.1997863247863247,
+      0.43162393162393164,
+      1.4770299145299146,
+      1.2561431623931625,
+      0.39129273504273504,
+      0.7922008547008547,
+      1.217147435897436,
+      1.2403846153846154,
+      1.001068376068376,
+      0.4497863247863248,
+      0.5056089743589743,
+      1.2919337606837606,
+      1.0430021367521367,
+      0.3779380341880342,
+      0.6132478632478633,
+      0.8800747863247863,
+      1.2251602564102564,
+      0.7385149572649573,
+      1.233440170940171,
+      0.9415064102564102,
+      0.5395299145299145,
+      1.0560897435897436,
+      0.5117521367521367,
+      1.4001068376068375,
+      0.5793269230769231,
+      1.0424679487179487,
+      0.41025641025641024,
+      0.2860576923076923,
+      0.49385683760683763,
+      0.8509615384615384,
+      1.0731837606837606,
+      1.3157051282051282,
+      0.6346153846153846,
+      0.5237713675213675,
+      0.4281517094017094,
+      0.4564636752136752,
+      0.4404380341880342,
+      0.9233440170940171,
+      1.141292735042735,
+      1.5745192307692308,
+      1.2091346153846154,
+      0.3560363247863248,
+      0.8397435897435898,
+      1.6418269230769231,
+      1.126068376068376,
+      0.5603632478632479,
+      1.3856837606837606,
+      0.3327991452991453,
+      0.8197115384615384,
+      0.7638888888888888,
+      0.9666132478632479,
+      0.7126068376068376,
+      0.6231303418803419,
+      1.138087606837607,
+      0.8824786324786325,
+      1.0504807692307692,
+      0.8961004273504274,
+      0.8838141025641025,
+      0.7358440170940171,
+      1.3215811965811965,
+      1.3790064102564104,
+      0.5144230769230769,
+      1.1965811965811965,
+      1.5739850427350428,
+      0.874198717948718,
+      1.5096153846153846,
+      1.3125,
+      1.1968482905982907,
+      1.3215811965811965,
+      1.0008012820512822,
+      1.0357905982905984,
+      1.3044871794871795,
+      0.657051282051282,
+      1.4895833333333333,
+      0.9647435897435898,
+      1.0584935897435896,
+      1.375534188034188,
+      0.5259081196581197,
+      0.8798076923076923,
+      1.4265491452991452,
+      0.8603098290598291,
+      0.8961004273504274,
+      1.264155982905983,
+      1.7278311965811965,
+      0.7248931623931624,
+      0.6813568376068376,
+      1.1963141025641026,
+      0.9340277777777778,
+      1.267094017094017,
+      0.7735042735042735,
+      0.3482905982905983
+    ],
+    "infer_probs": [
+      0.0016002127930928277,
+      0.004248635137838182,
+      0.008052551228665368,
+      0.012889206956153943,
+      0.018223536686095124,
+      0.02366250976912199
+    ],
+    "infer_info_gains": [
+      1.029851769666868,
+      0.6792945912039468,
+      0.4790693677797695,
+      0.3435310274516444,
+      0.25245394635844937
+    ],
+    "infer_delta_probs": [
+      0.002648422344745354,
+      0.003803916090827186,
+      0.004836655727488575,
+      0.005334329729941181,
+      0.0054389730830268655
+    ],
+    "infer_expected_reward_accs": [
+      0.40384615384615385,
+      0.7740384615384616,
+      0.9423076923076923,
+      0.9903846153846154,
+      0.9983974358974359
+    ],
+    "infer_expected_l2s": [
+      1.356186067872485,
+      0.9784815134542595,
+      0.814474543985168,
+      0.7224320185019728,
+      0.6628382416498068,
+      0.6212289369176899
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5472115384615384,
+      0.62625,
+      0.6655448717948713,
+      0.6944551282051282,
+      0.7139102564102566
+    ],
+    "infer_expected_random_set_idxs": [
+      0.8811431623931624,
+      1.0568910256410255,
+      1.4885149572649572,
+      1.0651709401709402,
+      0.9527243589743589,
+      1.2393162393162394,
+      1.1813568376068375,
+      0.7833867521367521,
+      1.3116987179487178,
+      1.3042200854700854,
+      0.6642628205128205,
+      1.264690170940171,
+      1.1984508547008548,
+      0.6404914529914529,
+      0.9802350427350427,
+      1.1821581196581197,
+      1.1020299145299146,
+      0.9602029914529915,
+      0.5018696581196581,
+      0.6113782051282052,
+      1.1487713675213675,
+      0.9925213675213675,
+      0.4751602564102564,
+      0.7323717948717948,
+      0.9121260683760684,
+      1.0435363247863247,
+      0.8047542735042735,
+      1.2139423076923077,
+      0.9212072649572649,
+      0.6968482905982906,
+      1.1519764957264957,
+      0.7427884615384616,
+      1.172809829059829,
+      0.7462606837606838,
+      0.96875,
+      0.8058226495726496,
+      0.6292735042735043,
+      0.6199252136752137,
+      0.9465811965811965,
+      1.0697115384615385,
+      1.1493055555555556,
+      0.6573183760683761,
+      0.6014957264957265,
+      0.7940705128205128,
+      0.6140491452991453,
+      0.6041666666666666,
+      0.9436431623931624,
+      1.1434294871794872,
+      1.5090811965811965,
+      1.1615918803418803,
+      0.6020299145299145,
+      0.9436431623931624,
+      1.4393696581196582,
+      1.1669337606837606,
+      0.6706730769230769,
+      1.1955128205128205,
+      0.5208333333333334,
+      0.7831196581196581,
+      0.8103632478632479,
+      0.9839743589743589,
+      0.7871260683760684,
+      1.1442307692307692,
+      1.143162393162393,
+      0.9481837606837606,
+      1.0798611111111112,
+      0.844284188034188,
+      0.9407051282051282,
+      0.6223290598290598,
+      1.3506944444444444,
+      1.3122329059829059,
+      0.6864316239316239,
+      1.093215811965812,
+      1.390224358974359,
+      0.9260149572649573,
+      1.3239850427350428,
+      1.1490384615384615,
+      1.1164529914529915,
+      1.2879273504273505,
+      0.8181089743589743,
+      0.9893162393162394,
+      1.2227564102564104,
+      0.6335470085470085,
+      1.3493589743589745,
+      1.0422008547008548,
+      1.0876068376068375,
+      1.1153846153846154,
+      0.6020299145299145,
+      0.8928952991452992,
+      1.296741452991453,
+      0.842948717948718,
+      0.6423611111111112,
+      1.2876602564102564,
+      1.5336538461538463,
+      0.7176816239316239,
+      0.7975427350427351,
+      1.0878739316239316,
+      0.9949252136752137,
+      1.1394230769230769,
+      0.7532051282051282,
+      0.5373931623931624
+    ],
+    "kls": [
+      0.0,
+      1.4337479767104648,
+      1.9680300243808209,
+      2.220121263988238,
+      2.3824528245799406,
+      2.4435614566475334
+    ],
+    "model_random_set_direct_accs": [
+      0.39943910256410226,
+      0.4929006410256414,
+      0.5368429487179489,
+      0.563958333333334,
+      0.582227564102564,
+      0.5985897435897434
+    ],
+    "model_random_set_direct_idxs": [
+      1.1535790598290598,
+      1.1757478632478633,
+      1.358440170940171,
+      1.3002136752136753,
+      1.047008547008547,
+      0.6607905982905983,
+      0.7665598290598291,
+      0.7529380341880342,
+      1.078525641025641,
+      1.5133547008547008,
+      1.0990918803418803,
+      0.8552350427350427,
+      1.3311965811965811,
+      0.9121260683760684,
+      1.4273504273504274,
+      1.439102564102564,
+      1.1984508547008548,
+      1.0114850427350428,
+      0.7550747863247863,
+      0.8870192307692307,
+      0.6696047008547008,
+      1.1701388888888888,
+      0.8095619658119658,
+      1.266025641025641,
+      0.9460470085470085,
+      0.8547008547008547,
+      0.8725961538461539,
+      1.3699252136752136,
+      0.7689636752136753,
+      1.2473290598290598,
+      1.3659188034188035,
+      1.325587606837607,
+      0.4703525641025641,
+      1.3352029914529915,
+      0.9030448717948718,
+      1.1169871794871795,
+      0.8338675213675214,
+      0.811698717948718,
+      1.2048611111111112,
+      0.7142094017094017,
+      1.2206196581196582,
+      1.2815170940170941,
+      0.7443910256410257,
+      1.1474358974358974,
+      1.0905448717948718,
+      0.6581196581196581,
+      0.8474893162393162,
+      1.3838141025641026,
+      1.2881944444444444,
+      1.0571581196581197,
+      1.218215811965812,
+      0.8653846153846154,
+      1.1917735042735043,
+      1.297809829059829,
+      1.1268696581196582,
+      1.2427884615384615,
+      0.8565705128205128,
+      0.718482905982906,
+      1.2999465811965811,
+      0.9829059829059829,
+      1.0731837606837606,
+      0.7793803418803419,
+      1.1535790598290598,
+      0.6290064102564102,
+      1.298344017094017,
+      0.9126602564102564,
+      0.9957264957264957,
+      0.48878205128205127,
+      0.8274572649572649,
+      0.7881944444444444,
+      1.0283119658119657,
+      0.6960470085470085,
+      1.2417200854700854,
+      0.6485042735042735,
+      1.1866987179487178,
+      0.7609508547008547,
+      0.7930021367521367,
+      1.0456730769230769,
+      0.8653846153846154,
+      1.1872329059829059,
+      1.1241987179487178,
+      0.8205128205128205,
+      1.076655982905983,
+      0.9732905982905983,
+      1.1944444444444444,
+      1.1153846153846154,
+      0.6861645299145299,
+      0.7449252136752137,
+      1.0830662393162394,
+      1.235042735042735,
+      0.5114850427350427,
+      1.0512820512820513,
+      1.2783119658119657,
+      1.1388888888888888,
+      1.3234508547008548,
+      0.6901709401709402,
+      0.9815705128205128,
+      0.4751602564102564,
+      0.5213675213675214,
+      0.750801282051282
+    ],
+    "model_expected_random_set_consistencies": [
+      0.34000000000000286,
+      0.7386057692307691,
+      0.763798076923077,
+      0.7629166666666658,
+      0.7681250000000002,
+      0.7721955128205122
+    ],
+    "model_random_set_consistencies": [
+      0.35543269230769375,
+      0.6934775641025644,
+      0.7173878205128207,
+      0.7161378205128203,
+      0.7258974358974359,
+      0.7276282051282047
+    ]
+  }
+}

results_v2/flight_uniform_kmin_38_seed2/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_38_seed2.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.6009615384615384,
+      0.6746794871794872,
+      0.7115384615384616,
+      0.7564102564102564,
+      0.7307692307692307
+    ],
+    "model_accs": [
+      0.6009615384615384,
+      0.6746794871794872,
+      0.7115384615384616,
+      0.7564102564102564,
+      0.7307692307692307
+    ],
+    "model_reward_accs": [
+      0.3766025641025641,
+      0.5801282051282052,
+      0.6778846153846154,
+      0.7467948717948718,
+      0.7435897435897436
+    ],
+    "model_consistencies": [
+      0.3974358974358974,
+      0.6506410256410257,
+      0.7323717948717948,
+      0.7451923076923077,
+      0.7483974358974359
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.0762206617999732,
+      0.9839357358741723,
+      0.8817629559852963,
+      0.8345441597530134,
+      0.8063090814914448
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.49331730769230786,
+      0.5433653846153847,
+      0.5894391025641026,
+      0.6140705128205135,
+      0.6262500000000001
+    ],
+    "model_random_set_idxs": [
+      0.7425213675213675,
+      1.1623931623931625,
+      1.328258547008547,
+      1.0993589743589745,
+      1.0299145299145298,
+      1.1525106837606838,
+      1.1036324786324787,
+      0.9722222222222222,
+      1.1006944444444444,
+      1.1282051282051282,
+      0.750801282051282,
+      1.0734508547008548,
+      1.2302350427350428,
+      0.8063568376068376,
+      1.0077457264957266,
+      1.110844017094017,
+      0.9115918803418803,
+      0.9236111111111112,
+      0.6856303418803419,
+      0.6444978632478633,
+      1.0582264957264957,
+      1.0120192307692308,
+      0.6712072649572649,
+      0.8461538461538461,
+      1.0681089743589745,
+      1.0355235042735043,
+      0.9270833333333334,
+      1.1645299145299146,
+      0.7799145299145299,
+      0.9420405982905983,
+      1.0539529914529915,
+      0.9823717948717948,
+      0.9513888888888888,
+      0.8979700854700855,
+      0.9417735042735043,
+      0.7841880341880342,
+      0.5184294871794872,
+      0.8530982905982906,
+      1.0916132478632479,
+      1.0184294871794872,
+      1.1025641025641026,
+      0.9479166666666666,
+      0.6501068376068376,
+      0.8298611111111112,
+      0.8656517094017094,
+      0.6386217948717948,
+      0.7569444444444444,
+      1.2809829059829059,
+      1.467681623931624,
+      1.0798611111111112,
+      0.8637820512820513,
+      1.080662393162393,
+      1.235844017094017,
+      1.2385149572649572,
+      0.6960470085470085,
+      1.25,
+      0.6225961538461539,
+      0.7972756410256411,
+      0.9647435897435898,
+      1.0972222222222223,
+      1.0125534188034189,
+      0.7740384615384616,
+      1.047542735042735,
+      1.0058760683760684,
+      1.3405448717948718,
+      1.0862713675213675,
+      1.0520833333333333,
+      0.6989850427350427,
+      1.0803952991452992,
+      1.1164529914529915,
+      0.7681623931623932,
+      0.9754273504273504,
+      1.2403846153846154,
+      0.8931623931623932,
+      1.2430555555555556,
+      1.0600961538461537,
+      1.0486111111111112,
+      1.1952457264957266,
+      1.0253739316239316,
+      1.1493055555555556,
+      1.0683760683760684,
+      0.7978098290598291,
+      1.2150106837606838,
+      1.2280982905982907,
+      0.9821047008547008,
+      1.1471688034188035,
+      0.7077991452991453,
+      0.78125,
+      1.1535790598290598,
+      1.014155982905983,
+      0.8533653846153846,
+      1.1642628205128205,
+      1.3856837606837606,
+      0.9650106837606838,
+      0.9642094017094017,
+      1.1183226495726495,
+      0.8878205128205128,
+      0.9260149572649573,
+      0.7518696581196581,
+      0.6228632478632479
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.003744505071819158,
+      0.0057038873831984725,
+      0.006095698211928935,
+      0.00898462511314663,
+      0.009285194892844227
+    ],
+    "model_info_gains": [
+      0.1131964568299756,
+      0.11711305863875854,
+      0.1908838753489445,
+      0.09482388514785223,
+      0.11168757530742657
+    ],
+    "model_delta_probs": [
+      0.0021444525727905748,
+      0.0019593823113793142,
+      0.00039181082873046346,
+      0.0028889269012176955,
+      0.0003005697796975955
+    ],
+    "model_expected_reward_accs": [
+      0.3798076923076923,
+      0.6474358974358975,
+      0.7163461538461539,
+      0.7948717948717948,
+      0.7676282051282052
+    ],
+    "model_expected_consistencies": [
+      0.3974358974358974,
+      0.6955128205128205,
+      0.7964743589743589,
+      0.8028846153846154,
+      0.7980769230769231
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.0548004202144479,
+      0.9490155700277788,
+      0.8546754994396255,
+      0.7969530667168697,
+      0.7649219675566776
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5063782051282049,
+      0.558653846153846,
+      0.6017948717948723,
+      0.6304807692307701,
+      0.6464262820512817
+    ],
+    "model_expected_random_set_idxs": [
+      0.9577991452991453,
+      1.2131410256410255,
+      1.279647435897436,
+      1.0769230769230769,
+      1.0336538461538463,
+      1.1124465811965811,
+      1.1794871794871795,
+      0.8971688034188035,
+      1.2064636752136753,
+      1.2628205128205128,
+      0.8485576923076923,
+      1.0977564102564104,
+      1.1829594017094016,
+      0.8482905982905983,
+      1.0128205128205128,
+      1.173878205128205,
+      0.9521901709401709,
+      0.9147970085470085,
+      0.6655982905982906,
+      0.6661324786324786,
+      1.106837606837607,
+      1.077190170940171,
+      0.6375534188034188,
+      0.8530982905982906,
+      1.0841346153846154,
+      1.0008012820512822,
+      0.9289529914529915,
+      1.1752136752136753,
+      0.7938034188034188,
+      0.9412393162393162,
+      1.0355235042735043,
+      0.9465811965811965,
+      0.9839743589743589,
+      0.9703525641025641,
+      0.9081196581196581,
+      0.8659188034188035,
+      0.6330128205128205,
+      0.8125,
+      1.1324786324786325,
+      1.0352564102564104,
+      1.0507478632478633,
+      0.9358974358974359,
+      0.6960470085470085,
+      0.9035790598290598,
+      0.8987713675213675,
+      0.6832264957264957,
+      0.7598824786324786,
+      1.2751068376068375,
+      1.4254807692307692,
+      1.0486111111111112,
+      0.8872863247863247,
+      1.0892094017094016,
+      1.1821581196581197,
+      1.1888354700854702,
+      0.6952457264957265,
+      1.234775641025641,
+      0.6688034188034188,
+      0.7783119658119658,
+      0.9073183760683761,
+      1.1047008547008548,
+      1.000534188034188,
+      1.1581196581196582,
+      1.0929487179487178,
+      1.0438034188034189,
+      1.2767094017094016,
+      0.9500534188034188,
+      1.0422008547008548,
+      0.5227029914529915,
+      1.0579594017094016,
+      1.1655982905982907,
+      0.7457264957264957,
+      1.0197649572649572,
+      1.2126068376068375,
+      0.9129273504273504,
+      1.218215811965812,
+      1.0320512820512822,
+      1.0590277777777777,
+      1.248397435897436,
+      0.8621794871794872,
+      1.1196581196581197,
+      1.1140491452991452,
+      0.782051282051282,
+      1.2470619658119657,
+      1.2190170940170941,
+      0.9487179487179487,
+      1.0886752136752136,
+      0.7059294871794872,
+      0.8018162393162394,
+      1.1570512820512822,
+      0.9286858974358975,
+      0.6506410256410257,
+      1.2008547008547008,
+      1.3720619658119657,
+      0.9345619658119658,
+      0.9316239316239316,
+      1.0432692307692308,
+      0.9358974358974359,
+      0.9919871794871795,
+      0.8202457264957265,
+      0.6153846153846154
+    ],
+    "infer_reward_accs": [
+      0.3766025641025641,
+      0.5272435897435898,
+      0.6602564102564102,
+      0.7035256410256411,
+      0.7964743589743589
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.151501351597007,
+      0.975323548915819,
+      0.8550391918574775,
+      0.7646750315546812,
+      0.7020411413299361
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.4623878205128206,
+      0.5496794871794872,
+      0.6060096153846154,
+      0.6475480769230773,
+      0.6751923076923076
+    ],
+    "infer_random_set_idxs": [
+      0.5200320512820513,
+      0.9754273504273504,
+      1.5966880341880343,
+      1.1055021367521367,
+      0.9447115384615384,
+      1.3982371794871795,
+      1.2748397435897436,
+      0.8525641025641025,
+      1.1917735042735043,
+      1.1701388888888888,
+      0.47489316239316237,
+      1.515758547008547,
+      1.2644230769230769,
+      0.3998397435897436,
+      0.8135683760683761,
+      1.2264957264957266,
+      1.2558760683760684,
+      0.9572649572649573,
+      0.4895833333333333,
+      0.5280448717948718,
+      1.2839209401709402,
+      1.0747863247863247,
+      0.3891559829059829,
+      0.6199252136752137,
+      0.9011752136752137,
+      1.220352564102564,
+      0.7897970085470085,
+      1.2385149572649572,
+      0.9233440170940171,
+      0.5526175213675214,
+      1.029647435897436,
+      0.5205662393162394,
+      1.4489850427350428,
+      0.5988247863247863,
+      1.0408653846153846,
+      0.45058760683760685,
+      0.30235042735042733,
+      0.5285790598290598,
+      0.8255876068376068,
+      1.0587606837606838,
+      1.3461538461538463,
+      0.6634615384615384,
+      0.5806623931623932,
+      0.452991452991453,
+      0.5037393162393162,
+      0.437232905982906,
+      0.9025106837606838,
+      1.1746794871794872,
+      1.5563568376068375,
+      1.2099358974358974,
+      0.3856837606837607,
+      0.8408119658119658,
+      1.6052350427350428,
+      1.1308760683760684,
+      0.593215811965812,
+      1.4500534188034189,
+      0.3317307692307692,
+      0.7796474358974359,
+      0.7142094017094017,
+      0.9399038461538461,
+      0.7486645299145299,
+      0.6228632478632479,
+      1.1690705128205128,
+      0.8709935897435898,
+      1.0373931623931625,
+      0.9297542735042735,
+      0.8309294871794872,
+      0.6842948717948718,
+      1.282852564102564,
+      1.3685897435897436,
+      0.5216346153846154,
+      1.202991452991453,
+      1.5320512820512822,
+      0.8488247863247863,
+      1.482905982905983,
+      1.3245192307692308,
+      1.1503739316239316,
+      1.34375,
+      1.0227029914529915,
+      1.1044337606837606,
+      1.2836538461538463,
+      0.7003205128205128,
+      1.4706196581196582,
+      0.9610042735042735,
+      1.0213675213675213,
+      1.3934294871794872,
+      0.5403311965811965,
+      0.8263888888888888,
+      1.4305555555555556,
+      0.8450854700854701,
+      0.9019764957264957,
+      1.2443910256410255,
+      1.7086004273504274,
+      0.7622863247863247,
+      0.6853632478632479,
+      1.2152777777777777,
+      0.9521901709401709,
+      1.265758547008547,
+      0.8058226495726496,
+      0.31490384615384615
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.004356828028466435,
+      0.008574899453716894,
+      0.013813574287805516,
+      0.019674009165007587,
+      0.025811868876842663
+    ],
+    "infer_info_gains": [
+      1.0540592337197794,
+      0.6903517283477237,
+      0.49475852588874614,
+      0.36179201675036005,
+      0.2705687507536941
+    ],
+    "infer_delta_probs": [
+      0.0027567755294378516,
+      0.004218071425250458,
+      0.005238674834088623,
+      0.005860434877202072,
+      0.006137859711835074
+    ],
+    "infer_expected_reward_accs": [
+      0.3798076923076923,
+      0.7756410256410257,
+      0.9198717948717948,
+      0.9727564102564102,
+      0.9935897435897436
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      0.9641988109829557,
+      0.8004775990365646,
+      0.6990257185148494,
+      0.6330973449988253,
+      0.5884732203260724
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5550160256410269,
+      0.6323237179487174,
+      0.6768269230769229,
+      0.7055128205128195,
+      0.7252884615384618
+    ],
+    "infer_expected_random_set_idxs": [
+      0.8928952991452992,
+      1.0699786324786325,
+      1.4890491452991452,
+      1.0694444444444444,
+      0.9457799145299145,
+      1.2371794871794872,
+      1.1770833333333333,
+      0.7799145299145299,
+      1.2532051282051282,
+      1.2705662393162394,
+      0.6939102564102564,
+      1.2793803418803418,
+      1.2048611111111112,
+      0.6781517094017094,
+      0.9919871794871795,
+      1.1872329059829059,
+      1.0753205128205128,
+      0.9743589743589743,
+      0.5197649572649573,
+      0.6022970085470085,
+      1.1129807692307692,
+      0.9906517094017094,
+      0.483974358974359,
+      0.7516025641025641,
+      0.9193376068376068,
+      1.064102564102564,
+      0.8068910256410257,
+      1.1848290598290598,
+      0.9177350427350427,
+      0.6915064102564102,
+      1.1498397435897436,
+      0.7369123931623932,
+      1.1896367521367521,
+      0.7612179487179487,
+      0.9847756410256411,
+      0.8039529914529915,
+      0.6826923076923077,
+      0.6447649572649573,
+      0.9214743589743589,
+      1.0229700854700854,
+      1.1276709401709402,
+      0.687232905982906,
+      0.625801282051282,
+      0.7935363247863247,
+      0.6311431623931624,
+      0.6383547008547008,
+      0.9476495726495726,
+      1.1346153846153846,
+      1.4775641025641026,
+      1.1367521367521367,
+      0.6100427350427351,
+      0.9358974358974359,
+      1.4262820512820513,
+      1.1976495726495726,
+      0.7139423076923077,
+      1.1968482905982907,
+      0.49919871794871795,
+      0.7847222222222222,
+      0.8210470085470085,
+      1.015758547008547,
+      0.8245192307692307,
+      1.172275641025641,
+      1.1274038461538463,
+      0.9233440170940171,
+      1.094818376068376,
+      0.8306623931623932,
+      0.9254807692307693,
+      0.6172542735042735,
+      1.3074252136752136,
+      1.3143696581196582,
+      0.7326388888888888,
+      1.0707799145299146,
+      1.3952991452991452,
+      0.9556623931623932,
+      1.3074252136752136,
+      1.1549145299145298,
+      1.0769230769230769,
+      1.2633547008547008,
+      0.7980769230769231,
+      0.9903846153846154,
+      1.232905982905983,
+      0.6658653846153846,
+      1.3298611111111112,
+      1.0737179487179487,
+      1.1022970085470085,
+      1.1503739316239316,
+      0.5886752136752137,
+      0.8955662393162394,
+      1.3127670940170941,
+      0.8509615384615384,
+      0.625534188034188,
+      1.251068376068376,
+      1.5138888888888888,
+      0.7334401709401709,
+      0.8135683760683761,
+      1.0961538461538463,
+      1.0518162393162394,
+      1.142628205128205,
+      0.7393162393162394,
+      0.5221688034188035
+    ],
+    "kls": [
+      0.0,
+      1.4602397456380845,
+      1.968786882844821,
+      2.172149124334424,
+      2.353224912847861,
+      2.4007642291555094
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.49693910256410273,
+      0.535849358974359,
+      0.5688782051282052,
+      0.5888141025641028,
+      0.6092788461538452
+    ],
+    "model_random_set_direct_idxs": [
+      1.1581196581196582,
+      1.0900106837606838,
+      1.4262820512820513,
+      1.3111645299145298,
+      1.0868055555555556,
+      0.6848290598290598,
+      0.7259615384615384,
+      0.7951388888888888,
+      1.0408653846153846,
+      1.500534188034188,
+      1.2166132478632479,
+      0.8597756410256411,
+      1.3450854700854702,
+      0.8851495726495726,
+      1.3776709401709402,
+      1.3918269230769231,
+      1.1781517094017093,
+      1.0373931623931625,
+      0.7166132478632479,
+      0.8587072649572649,
+      0.65625,
+      1.1615918803418803,
+      0.7860576923076923,
+      1.1877670940170941,
+      0.8691239316239316,
+      0.8771367521367521,
+      1.0032051282051282,
+      1.3258547008547008,
+      0.8210470085470085,
+      1.1626602564102564,
+      1.3517628205128205,
+      1.2280982905982907,
+      0.4895833333333333,
+      1.2769764957264957,
+      0.8859508547008547,
+      1.0729166666666667,
+      0.8357371794871795,
+      0.8162393162393162,
+      1.0032051282051282,
+      0.7620192307692307,
+      1.2136752136752136,
+      1.2096688034188035,
+      0.6981837606837606,
+      1.0753205128205128,
+      1.0520833333333333,
+      0.6685363247863247,
+      0.8448183760683761,
+      1.3082264957264957,
+      1.3060897435897436,
+      1.0552884615384615,
+      1.1637286324786325,
+      0.7801816239316239,
+      1.2638888888888888,
+      1.2793803418803418,
+      1.141559829059829,
+      1.2077991452991452,
+      0.7972756410256411,
+      0.7451923076923077,
+      1.3018162393162394,
+      1.0269764957264957,
+      1.0731837606837606,
+      0.8143696581196581,
+      1.1444978632478633,
+      0.6666666666666666,
+      1.2847222222222223,
+      0.9046474358974359,
+      1.0061431623931625,
+      0.5251068376068376,
+      0.8400106837606838,
+      1.1591880341880343,
+      1.0683760683760684,
+      0.6856303418803419,
+      1.2991452991452992,
+      0.8568376068376068,
+      1.2542735042735043,
+      0.7606837606837606,
+      0.7913995726495726,
+      1.0360576923076923,
+      0.8293269230769231,
+      1.1599893162393162,
+      1.1957799145299146,
+      0.7753739316239316,
+      1.0547542735042734,
+      1.0117521367521367,
+      1.2083333333333333,
+      1.1036324786324787,
+      0.6423611111111112,
+      0.8338675213675214,
+      1.106837606837607,
+      1.1848290598290598,
+      0.4719551282051282,
+      1.0836004273504274,
+      1.3675213675213675,
+      1.0945512820512822,
+      1.297809829059829,
+      0.7214209401709402,
+      1.0040064102564104,
+      0.5042735042735043,
+      0.4735576923076923,
+      0.7158119658119658
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7433653846153845,
+      0.7590705128205127,
+      0.7598878205128204,
+      0.7672435897435899,
+      0.7709935897435896
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.6910256410256418,
+      0.7171634615384618,
+      0.7136698717948715,
+      0.718942307692308,
+      0.7269711538461543
+    ]
+  }
+}

results_v2/flight_uniform_kmin_46_seed2/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_46_seed2.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.4375,
+      0.5256410256410257,
+      0.5865384615384616,
+      0.6057692307692307,
+      0.6266025641025641
+    ],
+    "model_accs": [
+      0.4375,
+      0.5256410256410257,
+      0.5865384615384616,
+      0.6057692307692307,
+      0.6266025641025641
+    ],
+    "model_reward_accs": [
+      0.3573717948717949,
+      0.5096153846153846,
+      0.5913461538461539,
+      0.5993589743589743,
+      0.6394230769230769
+    ],
+    "model_consistencies": [
+      0.358974358974359,
+      0.6410256410256411,
+      0.6794871794871795,
+      0.6746794871794872,
+      0.7067307692307693
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.087458303340424,
+      0.970389421878177,
+      0.9272960543071875,
+      0.880038868878741,
+      0.8582640091402732
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.4891666666666673,
+      0.5475641025641028,
+      0.5678044871794875,
+      0.5882852564102564,
+      0.5965064102564104
+    ],
+    "model_random_set_idxs": [
+      0.7323717948717948,
+      1.2128739316239316,
+      1.2727029914529915,
+      1.0833333333333333,
+      1.014957264957265,
+      1.110844017094017,
+      1.1487713675213675,
+      0.9794337606837606,
+      1.109775641025641,
+      1.1121794871794872,
+      0.8044871794871795,
+      1.0657051282051282,
+      1.2139423076923077,
+      0.8175747863247863,
+      1.0534188034188035,
+      1.1463675213675213,
+      0.8942307692307693,
+      0.9172008547008547,
+      0.7136752136752137,
+      0.6722756410256411,
+      1.0889423076923077,
+      0.9794337606837606,
+      0.7176816239316239,
+      0.9163995726495726,
+      1.1343482905982907,
+      1.032852564102564,
+      0.9428418803418803,
+      1.1677350427350428,
+      0.7545405982905983,
+      0.9778311965811965,
+      1.077991452991453,
+      1.016025641025641,
+      0.9174679487179487,
+      0.9391025641025641,
+      0.9823717948717948,
+      0.7897970085470085,
+      0.5285790598290598,
+      0.8565705128205128,
+      1.1466346153846154,
+      0.9759615384615384,
+      1.0972222222222223,
+      0.9901175213675214,
+      0.6848290598290598,
+      0.8493589743589743,
+      0.906517094017094,
+      0.6514423076923077,
+      0.7411858974358975,
+      1.3146367521367521,
+      1.4444444444444444,
+      1.079059829059829,
+      0.8872863247863247,
+      1.1450320512820513,
+      1.1805555555555556,
+      1.265491452991453,
+      0.6677350427350427,
+      1.2833867521367521,
+      0.6378205128205128,
+      0.8138354700854701,
+      0.9748931623931624,
+      1.09375,
+      0.9823717948717948,
+      0.7385149572649573,
+      1.013087606837607,
+      1.03125,
+      1.375,
+      1.0860042735042734,
+      1.108974358974359,
+      0.6808226495726496,
+      1.0969551282051282,
+      1.092681623931624,
+      0.7751068376068376,
+      1.0144230769230769,
+      1.1850961538461537,
+      0.8872863247863247,
+      1.173344017094017,
+      1.0550213675213675,
+      1.0531517094017093,
+      1.1765491452991452,
+      1.0606303418803418,
+      1.1917735042735043,
+      1.0208333333333333,
+      0.8181089743589743,
+      1.232905982905983,
+      1.2382478632478633,
+      0.9719551282051282,
+      1.1530448717948718,
+      0.717948717948718,
+      0.7630876068376068,
+      1.1378205128205128,
+      1.0325854700854702,
+      0.8766025641025641,
+      1.1442307692307692,
+      1.3659188034188035,
+      0.9967948717948718,
+      0.999732905982906,
+      1.0825320512820513,
+      0.8733974358974359,
+      0.9172008547008547,
+      0.7823183760683761,
+      0.6477029914529915
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.0034643283477964705,
+      0.005188672353807005,
+      0.005919051801513502,
+      0.007927802834353624,
+      0.007492936068611373
+    ],
+    "model_info_gains": [
+      0.17547253211244207,
+      0.23771486753090237,
+      0.06999899861296963,
+      0.12463588842061124,
+      0.030768091885622375
+    ],
+    "model_delta_probs": [
+      0.0018642758487678872,
+      0.0017243440060105341,
+      0.0007303794477064976,
+      0.0020087510328401217,
+      -0.00043486676574225124
+    ],
+    "model_expected_reward_accs": [
+      0.36698717948717946,
+      0.5448717948717948,
+      0.6137820512820513,
+      0.6698717948717948,
+      0.6682692307692307
+    ],
+    "model_expected_consistencies": [
+      0.3541666666666667,
+      0.6971153846153846,
+      0.7307692307692307,
+      0.7532051282051282,
+      0.7596153846153846
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.065093768964942,
+      0.9358625252950364,
+      0.8875634705422143,
+      0.8341417519496422,
+      0.8102335187116457
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5007532051282051,
+      0.5646794871794862,
+      0.5856089743589749,
+      0.611041666666666,
+      0.6194551282051278
+    ],
+    "model_expected_random_set_idxs": [
+      0.9708867521367521,
+      1.2561431623931625,
+      1.2302350427350428,
+      1.0422008547008548,
+      1.029647435897436,
+      1.1009615384615385,
+      1.2083333333333333,
+      0.9172008547008547,
+      1.2262286324786325,
+      1.2478632478632479,
+      0.9035790598290598,
+      1.0924145299145298,
+      1.1808226495726495,
+      0.8608440170940171,
+      1.0563568376068375,
+      1.1896367521367521,
+      0.9321581196581197,
+      0.9161324786324786,
+      0.6947115384615384,
+      0.7040598290598291,
+      1.1378205128205128,
+      1.0550213675213675,
+      0.6933760683760684,
+      0.9169337606837606,
+      1.1356837606837606,
+      0.9861111111111112,
+      0.9537927350427351,
+      1.171474358974359,
+      0.7614850427350427,
+      0.9861111111111112,
+      1.0582264957264957,
+      0.9762286324786325,
+      0.9610042735042735,
+      1.0080128205128205,
+      0.9399038461538461,
+      0.8707264957264957,
+      0.6466346153846154,
+      0.7983440170940171,
+      1.1784188034188035,
+      0.9714209401709402,
+      1.0651709401709402,
+      0.9604700854700855,
+      0.7489316239316239,
+      0.9284188034188035,
+      0.9476495726495726,
+      0.686698717948718,
+      0.7441239316239316,
+      1.2946047008547008,
+      1.3971688034188035,
+      1.0200320512820513,
+      0.8894230769230769,
+      1.1362179487179487,
+      1.1442307692307692,
+      1.2152777777777777,
+      0.655448717948718,
+      1.2729700854700854,
+      0.7174145299145299,
+      0.7897970085470085,
+      0.937232905982906,
+      1.1212606837606838,
+      0.9572649572649573,
+      1.1340811965811965,
+      1.0769230769230769,
+      1.049412393162393,
+      1.3213141025641026,
+      0.9524572649572649,
+      1.0998931623931625,
+      0.49278846153846156,
+      1.0611645299145298,
+      1.1359508547008548,
+      0.7435897435897436,
+      1.0721153846153846,
+      1.1626602564102564,
+      0.9172008547008547,
+      1.1450320512820513,
+      1.0264423076923077,
+      1.0592948717948718,
+      1.2572115384615385,
+      0.8771367521367521,
+      1.1599893162393162,
+      1.0822649572649572,
+      0.8191773504273504,
+      1.2694978632478633,
+      1.2232905982905984,
+      0.9177350427350427,
+      1.092681623931624,
+      0.7203525641025641,
+      0.8015491452991453,
+      1.1298076923076923,
+      0.9417735042735043,
+      0.6412927350427351,
+      1.1842948717948718,
+      1.3477564102564104,
+      0.9714209401709402,
+      0.967948717948718,
+      0.9778311965811965,
+      0.9220085470085471,
+      0.969551282051282,
+      0.8733974358974359,
+      0.6428952991452992
+    ],
+    "infer_reward_accs": [
+      0.3541666666666667,
+      0.5128205128205128,
+      0.5592948717948718,
+      0.7035256410256411,
+      0.780448717948718
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.1427710365748351,
+      0.9720881500466481,
+      0.8177517271759489,
+      0.712411341060424,
+      0.646890327764258
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.46540064102564077,
+      0.5509134615384619,
+      0.6258814102564109,
+      0.6752403846153852,
+      0.7040064102564104
+    ],
+    "infer_random_set_idxs": [
+      0.5547542735042735,
+      0.9797008547008547,
+      1.580128205128205,
+      1.0902777777777777,
+      0.9391025641025641,
+      1.3319978632478633,
+      1.2764423076923077,
+      0.8496260683760684,
+      1.2200854700854702,
+      1.1797542735042734,
+      0.5114850427350427,
+      1.4508547008547008,
+      1.2467948717948718,
+      0.42895299145299143,
+      0.8263888888888888,
+      1.220352564102564,
+      1.217147435897436,
+      0.9770299145299145,
+      0.47836538461538464,
+      0.5459401709401709,
+      1.266025641025641,
+      1.0400641025641026,
+      0.42628205128205127,
+      0.6503739316239316,
+      0.9246794871794872,
+      1.2152777777777777,
+      0.7833867521367521,
+      1.2331730769230769,
+      0.9217414529914529,
+      0.5974893162393162,
+      1.0916132478632479,
+      0.5753205128205128,
+      1.3803418803418803,
+      0.6367521367521367,
+      1.0315170940170941,
+      0.4636752136752137,
+      0.3469551282051282,
+      0.5422008547008547,
+      0.8557692307692307,
+      1.0360576923076923,
+      1.3034188034188035,
+      0.7024572649572649,
+      0.592948717948718,
+      0.4893162393162393,
+      0.5272435897435898,
+      0.49198717948717946,
+      0.9206730769230769,
+      1.1522435897435896,
+      1.532852564102564,
+      1.1856303418803418,
+      0.4479166666666667,
+      0.8715277777777778,
+      1.5798611111111112,
+      1.1487713675213675,
+      0.6006944444444444,
+      1.4025106837606838,
+      0.38274572649572647,
+      0.8167735042735043,
+      0.7681623931623932,
+      0.9607371794871795,
+      0.7302350427350427,
+      0.6290064102564102,
+      1.1436965811965811,
+      0.8822115384615384,
+      1.0924145299145298,
+      0.9105235042735043,
+      0.8950320512820513,
+      0.7110042735042735,
+      1.2799145299145298,
+      1.3167735042735043,
+      0.5766559829059829,
+      1.1989850427350428,
+      1.5243055555555556,
+      0.8619123931623932,
+      1.4409722222222223,
+      1.2903311965811965,
+      1.1493055555555556,
+      1.3015491452991452,
+      1.0403311965811965,
+      1.0707799145299146,
+      1.2783119658119657,
+      0.7059294871794872,
+      1.4297542735042734,
+      0.9612713675213675,
+      1.045405982905983,
+      1.3798076923076923,
+      0.5793269230769231,
+      0.8482905982905983,
+      1.4033119658119657,
+      0.8824786324786325,
+      0.9051816239316239,
+      1.2134081196581197,
+      1.6642628205128205,
+      0.780448717948718,
+      0.7077991452991453,
+      1.1810897435897436,
+      0.9313568376068376,
+      1.2427884615384615,
+      0.8034188034188035,
+      0.3683226495726496
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.004637383858467869,
+      0.009826805149679365,
+      0.017446453089481413,
+      0.026058936861306555,
+      0.03533104713549106
+    ],
+    "infer_info_gains": [
+      1.102690994471146,
+      0.7743494057379064,
+      0.5792868416204681,
+      0.41050487274756847,
+      0.300437605506844
+    ],
+    "infer_delta_probs": [
+      0.003037331359439285,
+      0.005189421291211497,
+      0.007619647939802048,
+      0.008612483771825143,
+      0.009272110274184505
+    ],
+    "infer_expected_reward_accs": [
+      0.36698717948717946,
+      0.6458333333333334,
+      0.8413461538461539,
+      0.9711538461538461,
+      0.9823717948717948
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      0.9530802090504897,
+      0.7724877556853894,
+      0.6583210070991964,
+      0.5851700692166012,
+      0.5389264355319853
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5565705128205131,
+      0.6443589743589739,
+      0.6994230769230766,
+      0.7324519230769229,
+      0.7524679487179492
+    ],
+    "infer_expected_random_set_idxs": [
+      0.9185363247863247,
+      1.0729166666666667,
+      1.4906517094017093,
+      1.0897435897435896,
+      0.9775641025641025,
+      1.203525641025641,
+      1.2000534188034189,
+      0.8135683760683761,
+      1.2454594017094016,
+      1.28125,
+      0.7248931623931624,
+      1.2379807692307692,
+      1.2069978632478633,
+      0.6701388888888888,
+      0.9871794871794872,
+      1.1907051282051282,
+      1.0563568376068375,
+      0.9743589743589743,
+      0.5221688034188035,
+      0.6089743589743589,
+      1.1167200854700854,
+      0.9874465811965812,
+      0.5197649572649573,
+      0.7588141025641025,
+      0.9230769230769231,
+      1.075587606837607,
+      0.8199786324786325,
+      1.1845619658119657,
+      0.9035790598290598,
+      0.7128739316239316,
+      1.1672008547008548,
+      0.7441239316239316,
+      1.1559829059829059,
+      0.7561431623931624,
+      0.9754273504273504,
+      0.7625534188034188,
+      0.6837606837606838,
+      0.6498397435897436,
+      0.9460470085470085,
+      1.016559829059829,
+      1.1145833333333333,
+      0.7256944444444444,
+      0.6450320512820513,
+      0.7743055555555556,
+      0.688301282051282,
+      0.6773504273504274,
+      0.9105235042735043,
+      1.1081730769230769,
+      1.4553952991452992,
+      1.0966880341880343,
+      0.6348824786324786,
+      0.9476495726495726,
+      1.3947649572649572,
+      1.1989850427350428,
+      0.6891025641025641,
+      1.1896367521367521,
+      0.5697115384615384,
+      0.8261217948717948,
+      0.8378739316239316,
+      1.0352564102564104,
+      0.8092948717948718,
+      1.171741452991453,
+      1.123931623931624,
+      0.9471153846153846,
+      1.139690170940171,
+      0.8509615384615384,
+      0.96875,
+      0.6137820512820513,
+      1.2857905982905984,
+      1.2825854700854702,
+      0.7425213675213675,
+      1.0878739316239316,
+      1.3961004273504274,
+      0.9727564102564102,
+      1.2788461538461537,
+      1.1153846153846154,
+      1.0876068376068375,
+      1.28125,
+      0.8050213675213675,
+      0.9759615384615384,
+      1.2411858974358974,
+      0.6792200854700855,
+      1.3210470085470085,
+      1.0742521367521367,
+      1.1049679487179487,
+      1.169871794871795,
+      0.6522435897435898,
+      0.9105235042735043,
+      1.3007478632478633,
+      0.8520299145299145,
+      0.6327457264957265,
+      1.2414529914529915,
+      1.5018696581196582,
+      0.7713675213675214,
+      0.8226495726495726,
+      1.0625,
+      1.0208333333333333,
+      1.1493055555555556,
+      0.7561431623931624,
+      0.5243055555555556
+    ],
+    "kls": [
+      0.0,
+      1.456193426765321,
+      2.092098631009541,
+      2.461672805766478,
+      2.653471205441637,
+      2.7433783559418106
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.47929487179487185,
+      0.5181089743589743,
+      0.545048076923077,
+      0.5637820512820516,
+      0.577628205128205
+    ],
+    "model_random_set_direct_idxs": [
+      1.1613247863247864,
+      1.1685363247863247,
+      1.4292200854700854,
+      1.3066239316239316,
+      1.0750534188034189,
+      0.6712072649572649,
+      0.7628205128205128,
+      0.7654914529914529,
+      1.0742521367521367,
+      1.5002670940170941,
+      1.2777777777777777,
+      0.8501602564102564,
+      1.361912393162393,
+      0.9185363247863247,
+      1.3995726495726495,
+      1.424412393162393,
+      1.1789529914529915,
+      1.0264423076923077,
+      0.7280982905982906,
+      0.8774038461538461,
+      0.6330128205128205,
+      1.1458333333333333,
+      0.8186431623931624,
+      1.2377136752136753,
+      0.9014423076923077,
+      0.8736645299145299,
+      1.0405982905982907,
+      1.327457264957265,
+      0.7692307692307693,
+      1.2126068376068375,
+      1.3872863247863247,
+      1.2751068376068375,
+      0.48370726495726496,
+      1.2825854700854702,
+      0.9019764957264957,
+      1.0726495726495726,
+      0.8637820512820513,
+      0.8167735042735043,
+      1.0357905982905984,
+      0.719017094017094,
+      1.2251602564102564,
+      1.2804487179487178,
+      0.719284188034188,
+      1.1063034188034189,
+      1.1060363247863247,
+      0.6495726495726496,
+      0.8226495726495726,
+      1.3354700854700854,
+      1.3068910256410255,
+      1.0080128205128205,
+      1.1821581196581197,
+      0.8146367521367521,
+      1.205128205128205,
+      1.3215811965811965,
+      1.1268696581196582,
+      1.2422542735042734,
+      0.843215811965812,
+      0.7561431623931624,
+      1.3159722222222223,
+      1.0112179487179487,
+      1.0584935897435896,
+      0.8170405982905983,
+      1.1538461538461537,
+      0.625534188034188,
+      1.3186431623931625,
+      0.9172008547008547,
+      1.0213675213675213,
+      0.4706196581196581,
+      0.7996794871794872,
+      1.1263354700854702,
+      1.0729166666666667,
+      0.7147435897435898,
+      1.295940170940171,
+      0.8530982905982906,
+      1.201121794871795,
+      0.7470619658119658,
+      0.7537393162393162,
+      1.0422008547008548,
+      0.8253205128205128,
+      1.1661324786324787,
+      1.1701388888888888,
+      0.8026175213675214,
+      1.0528846153846154,
+      0.9946581196581197,
+      1.1901709401709402,
+      1.1228632478632479,
+      0.6720085470085471,
+      0.8010149572649573,
+      1.080662393162393,
+      1.2208867521367521,
+      0.47863247863247865,
+      1.0625,
+      1.2831196581196582,
+      1.1471688034188035,
+      1.3146367521367521,
+      0.6634615384615384,
+      0.9927884615384616,
+      0.49946581196581197,
+      0.5157585470085471,
+      0.7259615384615384
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7316506410256404,
+      0.7410737179487176,
+      0.7532852564102561,
+      0.7523878205128202,
+      0.7572756410256417
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.6798076923076929,
+      0.6891666666666674,
+      0.7054967948717944,
+      0.7046153846153841,
+      0.7142948717948719
+    ]
+  }
+}

results_v2/flight_uniform_kmin_55_seed1/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_55_seed1.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.3108974358974359,
+      0.4310897435897436,
+      0.5048076923076923,
+      0.532051282051282,
+      0.5592948717948718
+    ],
+    "model_accs": [
+      0.3108974358974359,
+      0.4310897435897436,
+      0.5048076923076923,
+      0.532051282051282,
+      0.5592948717948718
+    ],
+    "model_reward_accs": [
+      0.34455128205128205,
+      0.453525641025641,
+      0.5032051282051282,
+      0.5625,
+      0.592948717948718
+    ],
+    "model_consistencies": [
+      0.3317307692307692,
+      0.6185897435897436,
+      0.6842948717948718,
+      0.7035256410256411,
+      0.6634615384615384
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.124508685030557,
+      1.0128510295132183,
+      0.9586535501714604,
+      0.8981525848463221,
+      0.8905827192207831
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.46908653846153936,
+      0.5256730769230772,
+      0.5554166666666662,
+      0.5804326923076926,
+      0.583301282051282
+    ],
+    "model_random_set_idxs": [
+      0.780448717948718,
+      1.1949786324786325,
+      1.2716346153846154,
+      1.1167200854700854,
+      1.063034188034188,
+      1.1148504273504274,
+      1.1159188034188035,
+      0.9893162393162394,
+      1.0873397435897436,
+      1.1006944444444444,
+      0.8036858974358975,
+      1.0480769230769231,
+      1.2540064102564104,
+      0.8317307692307693,
+      1.044871794871795,
+      1.1047008547008548,
+      0.8856837606837606,
+      0.8952991452991453,
+      0.7027243589743589,
+      0.6629273504273504,
+      1.0400641025641026,
+      1.0189636752136753,
+      0.7040598290598291,
+      0.8870192307692307,
+      1.1180555555555556,
+      0.9938568376068376,
+      0.9452457264957265,
+      1.140224358974359,
+      0.7473290598290598,
+      1.0082799145299146,
+      1.0809294871794872,
+      1.0411324786324787,
+      0.9022435897435898,
+      0.9473824786324786,
+      0.9188034188034188,
+      0.8311965811965812,
+      0.5360576923076923,
+      0.8926282051282052,
+      1.123931623931624,
+      0.9981303418803419,
+      1.0852029914529915,
+      1.0192307692307692,
+      0.6776175213675214,
+      0.8816773504273504,
+      0.9284188034188035,
+      0.6826923076923077,
+      0.7286324786324786,
+      1.297275641025641,
+      1.4452457264957266,
+      1.0651709401709402,
+      0.937232905982906,
+      1.1279380341880343,
+      1.172542735042735,
+      1.2735042735042734,
+      0.7150106837606838,
+      1.2620192307692308,
+      0.6760149572649573,
+      0.8106303418803419,
+      0.9895833333333334,
+      1.1028311965811965,
+      1.0408653846153846,
+      0.7764423076923077,
+      0.9925213675213675,
+      1.0189636752136753,
+      1.4038461538461537,
+      1.0854700854700854,
+      1.0852029914529915,
+      0.7061965811965812,
+      1.0293803418803418,
+      1.0574252136752136,
+      0.7900641025641025,
+      0.9743589743589743,
+      1.1957799145299146,
+      0.8907585470085471,
+      1.203792735042735,
+      1.0416666666666667,
+      1.0435363247863247,
+      1.1760149572649572,
+      1.0446047008547008,
+      1.1762820512820513,
+      1.0432692307692308,
+      0.811965811965812,
+      1.2000534188034189,
+      1.2267628205128205,
+      0.9783653846153846,
+      1.1447649572649572,
+      0.749465811965812,
+      0.7604166666666666,
+      1.1367521367521367,
+      1.0520833333333333,
+      0.8555021367521367,
+      1.1282051282051282,
+      1.3477564102564104,
+      1.016559829059829,
+      0.9890491452991453,
+      1.09375,
+      0.8472222222222222,
+      0.8966346153846154,
+      0.7721688034188035,
+      0.6503739316239316
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.00275958107991815,
+      0.003989658218264853,
+      0.005192947231174791,
+      0.006251950800961372,
+      0.006371420299715124
+    ],
+    "model_info_gains": [
+      0.227254565772049,
+      0.040634409077534336,
+      0.11545044892848494,
+      0.09742612242343342,
+      0.08574706112538175
+    ],
+    "model_delta_probs": [
+      0.0011595285808895666,
+      0.0012300771383467034,
+      0.0012032890129099372,
+      0.0010590035697865812,
+      0.00011946949875375222
+    ],
+    "model_expected_reward_accs": [
+      0.35096153846153844,
+      0.4551282051282051,
+      0.5416666666666666,
+      0.5769230769230769,
+      0.6073717948717948
+    ],
+    "model_expected_consistencies": [
+      0.34134615384615385,
+      0.6923076923076923,
+      0.7339743589743589,
+      0.7419871794871795,
+      0.7275641025641025
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.0895212597317192,
+      0.9773295138908689,
+      0.9160239667724889,
+      0.8660583641751157,
+      0.834543134699745
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.4857211538461533,
+      0.5440224358974356,
+      0.5752884615384622,
+      0.5971794871794879,
+      0.6124519230769239
+    ],
+    "model_expected_random_set_idxs": [
+      1.0315170940170941,
+      1.2403846153846154,
+      1.2387820512820513,
+      1.0643696581196582,
+      1.0854700854700854,
+      1.094284188034188,
+      1.1896367521367521,
+      0.9428418803418803,
+      1.1933760683760684,
+      1.2534722222222223,
+      0.9123931623931624,
+      1.0753205128205128,
+      1.2075320512820513,
+      0.8856837606837606,
+      1.0235042735042734,
+      1.1549145299145298,
+      0.9233440170940171,
+      0.874198717948718,
+      0.6912393162393162,
+      0.6861645299145299,
+      1.1038995726495726,
+      1.0897435897435896,
+      0.6939102564102564,
+      0.8725961538461539,
+      1.1052350427350428,
+      0.9452457264957265,
+      0.9428418803418803,
+      1.1279380341880343,
+      0.7580128205128205,
+      1.0184294871794872,
+      1.0510149572649572,
+      1.000534188034188,
+      0.9492521367521367,
+      1.0253739316239316,
+      0.8774038461538461,
+      0.9078525641025641,
+      0.6485042735042735,
+      0.8461538461538461,
+      1.1626602564102564,
+      1.0056089743589745,
+      1.0387286324786325,
+      1.0109508547008548,
+      0.7353098290598291,
+      0.969284188034188,
+      0.9676816239316239,
+      0.7286324786324786,
+      0.7417200854700855,
+      1.266559829059829,
+      1.3936965811965811,
+      0.9853098290598291,
+      0.9577991452991453,
+      1.1188568376068375,
+      1.1292735042735043,
+      1.1931089743589745,
+      0.7061965811965812,
+      1.2582799145299146,
+      0.7572115384615384,
+      0.7932692307692307,
+      0.9444444444444444,
+      1.1081730769230769,
+      1.0202991452991452,
+      1.1669337606837606,
+      1.0654380341880343,
+      1.0691773504273505,
+      1.342147435897436,
+      0.9409722222222222,
+      1.0769230769230769,
+      0.5048076923076923,
+      1.0221688034188035,
+      1.0905448717948718,
+      0.7588141025641025,
+      1.0184294871794872,
+      1.1655982905982907,
+      0.9123931623931624,
+      1.1984508547008548,
+      1.0114850427350428,
+      1.0566239316239316,
+      1.2478632478632479,
+      0.8506944444444444,
+      1.1495726495726495,
+      1.1290064102564104,
+      0.8066239316239316,
+      1.2206196581196582,
+      1.1981837606837606,
+      0.9278846153846154,
+      1.0849358974358974,
+      0.7422542735042735,
+      0.7991452991452992,
+      1.1228632478632479,
+      0.9540598290598291,
+      0.6143162393162394,
+      1.1834935897435896,
+      1.3092948717948718,
+      0.9855769230769231,
+      0.9529914529914529,
+      0.9949252136752137,
+      0.9169337606837606,
+      0.9428418803418803,
+      0.8723290598290598,
+      0.6423611111111112
+    ],
+    "infer_reward_accs": [
+      0.34455128205128205,
+      0.4567307692307692,
+      0.5144230769230769,
+      0.6217948717948718,
+      0.7435897435897436
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.129183104735781,
+      0.9387228817003114,
+      0.7594297039283665,
+      0.624045302270574,
+      0.5503680865748208
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.47269230769230774,
+      0.5670352564102573,
+      0.6525160256410262,
+      0.7136378205128208,
+      0.7441185897435899
+    ],
+    "infer_random_set_idxs": [
+      0.5865384615384616,
+      0.9965277777777778,
+      1.5523504273504274,
+      1.1258012820512822,
+      0.9698183760683761,
+      1.3338675213675213,
+      1.2278311965811965,
+      0.8536324786324786,
+      1.1931089743589745,
+      1.205662393162393,
+      0.5408653846153846,
+      1.4059829059829059,
+      1.2614850427350428,
+      0.4639423076923077,
+      0.8488247863247863,
+      1.2102029914529915,
+      1.202457264957265,
+      1.0114850427350428,
+      0.49278846153846156,
+      0.5472756410256411,
+      1.2366452991452992,
+      1.0499465811965811,
+      0.4375,
+      0.6495726495726496,
+      0.90625,
+      1.200587606837607,
+      0.7948717948717948,
+      1.2291666666666667,
+      0.905715811965812,
+      0.6209935897435898,
+      1.076655982905983,
+      0.6266025641025641,
+      1.3338675213675213,
+      0.6533119658119658,
+      1.016025641025641,
+      0.4906517094017094,
+      0.37927350427350426,
+      0.5662393162393162,
+      0.8629807692307693,
+      1.063034188034188,
+      1.2879273504273505,
+      0.7294337606837606,
+      0.5924145299145299,
+      0.5237713675213675,
+      0.5411324786324786,
+      0.5178952991452992,
+      0.8974358974358975,
+      1.1487713675213675,
+      1.5365918803418803,
+      1.1789529914529915,
+      0.4580662393162393,
+      0.874465811965812,
+      1.5422008547008548,
+      1.1642628205128205,
+      0.625267094017094,
+      1.361912393162393,
+      0.42414529914529914,
+      0.8181089743589743,
+      0.8018162393162394,
+      0.9703525641025641,
+      0.7780448717948718,
+      0.6674679487179487,
+      1.1463675213675213,
+      0.8766025641025641,
+      1.0894764957264957,
+      0.9310897435897436,
+      0.8800747863247863,
+      0.7139423076923077,
+      1.2393162393162394,
+      1.3015491452991452,
+      0.6065705128205128,
+      1.170405982905983,
+      1.5024038461538463,
+      0.8859508547008547,
+      1.4564636752136753,
+      1.2478632478632479,
+      1.1185897435897436,
+      1.28125,
+      1.0008012820512822,
+      1.0507478632478633,
+      1.2620192307692308,
+      0.7045940170940171,
+      1.4113247863247864,
+      0.9591346153846154,
+      1.0368589743589745,
+      1.3221153846153846,
+      0.6063034188034188,
+      0.8739316239316239,
+      1.374465811965812,
+      0.9083867521367521,
+      0.8643162393162394,
+      1.2155448717948718,
+      1.6493055555555556,
+      0.8079594017094017,
+      0.7612179487179487,
+      1.1909722222222223,
+      0.9350961538461539,
+      1.2048611111111112,
+      0.7841880341880342,
+      0.4043803418803419
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.004760087434696731,
+      0.010798990678127805,
+      0.021116543487947545,
+      0.03356260601760415,
+      0.04824681067531436
+    ],
+    "infer_info_gains": [
+      1.1400535044272428,
+      0.8742282505432604,
+      0.6771844309141956,
+      0.486675661451798,
+      0.36165317452191253
+    ],
+    "infer_delta_probs": [
+      0.003160034935668148,
+      0.006038903243431075,
+      0.010317552809819738,
+      0.012446062529656606,
+      0.014684204657710209
+    ],
+    "infer_expected_reward_accs": [
+      0.35096153846153844,
+      0.5256410256410257,
+      0.7355769230769231,
+      0.9022435897435898,
+      0.9647435897435898
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      0.9473546011946922,
+      0.7354654735280283,
+      0.600495458452954,
+      0.5124086369226976,
+      0.464256459219298
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5623878205128211,
+      0.6640224358974363,
+      0.7260897435897435,
+      0.764919871794872,
+      0.7864743589743589
+    ],
+    "infer_expected_random_set_idxs": [
+      0.8824786324786325,
+      1.0670405982905984,
+      1.4826388888888888,
+      1.0689102564102564,
+      0.9615384615384616,
+      1.1981837606837606,
+      1.189102564102564,
+      0.7999465811965812,
+      1.2686965811965811,
+      1.2887286324786325,
+      0.7270299145299145,
+      1.2043269230769231,
+      1.1915064102564104,
+      0.6781517094017094,
+      0.9837072649572649,
+      1.1848290598290598,
+      1.0600961538461537,
+      1.0144230769230769,
+      0.5170940170940171,
+      0.6303418803418803,
+      1.1295405982905984,
+      0.9759615384615384,
+      0.5090811965811965,
+      0.7564102564102564,
+      0.9257478632478633,
+      1.0657051282051282,
+      0.8079594017094017,
+      1.203525641025641,
+      0.9249465811965812,
+      0.7208867521367521,
+      1.1637286324786325,
+      0.7601495726495726,
+      1.1303418803418803,
+      0.7686965811965812,
+      0.9786324786324786,
+      0.7529380341880342,
+      0.6669337606837606,
+      0.6404914529914529,
+      0.969017094017094,
+      1.0411324786324787,
+      1.1322115384615385,
+      0.7286324786324786,
+      0.6303418803418803,
+      0.7737713675213675,
+      0.6650641025641025,
+      0.6674679487179487,
+      0.9073183760683761,
+      1.1038995726495726,
+      1.4569978632478633,
+      1.1145833333333333,
+      0.624465811965812,
+      0.9476495726495726,
+      1.4107905982905984,
+      1.2131410256410255,
+      0.6842948717948718,
+      1.1776175213675213,
+      0.5665064102564102,
+      0.8186431623931624,
+      0.8413461538461539,
+      1.0186965811965811,
+      0.8261217948717948,
+      1.1340811965811965,
+      1.1436965811965811,
+      0.9193376068376068,
+      1.1220619658119657,
+      0.8592414529914529,
+      0.9700854700854701,
+      0.6036324786324786,
+      1.295405982905983,
+      1.2855235042735043,
+      0.7435897435897436,
+      1.0611645299145298,
+      1.3774038461538463,
+      0.9668803418803419,
+      1.2841880341880343,
+      1.107905982905983,
+      1.0990918803418803,
+      1.2449252136752136,
+      0.8074252136752137,
+      0.9618055555555556,
+      1.2238247863247864,
+      0.655448717948718,
+      1.3026175213675213,
+      1.0838675213675213,
+      1.0822649572649572,
+      1.1487713675213675,
+      0.6372863247863247,
+      0.9313568376068376,
+      1.2604166666666667,
+      0.8274572649572649,
+      0.6354166666666666,
+      1.2291666666666667,
+      1.484241452991453,
+      0.7911324786324786,
+      0.8247863247863247,
+      1.0857371794871795,
+      1.0216346153846154,
+      1.1274038461538463,
+      0.7454594017094017,
+      0.5587606837606838
+    ],
+    "kls": [
+      0.0,
+      1.537228239419718,
+      2.348643398677034,
+      2.869194699496746,
+      3.138830316308671,
+      3.2847228687044563
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.46474358974358937,
+      0.4979967948717952,
+      0.5246955128205129,
+      0.5451121794871792,
+      0.5596634615384616
+    ],
+    "model_random_set_direct_idxs": [
+      1.2184829059829059,
+      1.1295405982905984,
+      1.4495192307692308,
+      1.3218482905982907,
+      1.1225961538461537,
+      0.6693376068376068,
+      0.7489316239316239,
+      0.7673611111111112,
+      1.0379273504273505,
+      1.4938568376068375,
+      1.2825854700854702,
+      0.844551282051282,
+      1.4041132478632479,
+      0.9310897435897436,
+      1.3808760683760684,
+      1.4145299145299146,
+      1.154647435897436,
+      1.032852564102564,
+      0.7240918803418803,
+      0.8779380341880342,
+      0.625,
+      1.1877670940170941,
+      0.7999465811965812,
+      1.1875,
+      0.8763354700854701,
+      0.8549679487179487,
+      1.015491452991453,
+      1.2911324786324787,
+      0.7796474358974359,
+      1.2403846153846154,
+      1.4011752136752136,
+      1.2540064102564104,
+      0.47889957264957267,
+      1.3018162393162394,
+      0.8768696581196581,
+      1.0961538461538463,
+      0.8450854700854701,
+      0.8290598290598291,
+      1.0371260683760684,
+      0.7310363247863247,
+      1.1746794871794872,
+      1.298344017094017,
+      0.719017094017094,
+      1.1185897435897436,
+      1.1159188034188035,
+      0.6450320512820513,
+      0.8076923076923077,
+      1.295405982905983,
+      1.2844551282051282,
+      0.9764957264957265,
+      1.2099358974358974,
+      0.7751068376068376,
+      1.2096688034188035,
+      1.3074252136752136,
+      1.1565170940170941,
+      1.235309829059829,
+      0.8819444444444444,
+      0.7406517094017094,
+      1.3060897435897436,
+      1.0232371794871795,
+      1.0862713675213675,
+      0.8034188034188035,
+      1.1511752136752136,
+      0.6324786324786325,
+      1.3362713675213675,
+      0.9153311965811965,
+      1.0,
+      0.4829059829059829,
+      0.7705662393162394,
+      1.079594017094017,
+      1.0627670940170941,
+      0.6917735042735043,
+      1.2935363247863247,
+      0.8392094017094017,
+      1.235844017094017,
+      0.7572115384615384,
+      0.7475961538461539,
+      0.999198717948718,
+      0.7922008547008547,
+      1.1583867521367521,
+      1.2099358974358974,
+      0.8074252136752137,
+      1.0438034188034189,
+      0.9882478632478633,
+      1.1992521367521367,
+      1.1185897435897436,
+      0.6808226495726496,
+      0.8036858974358975,
+      1.0966880341880343,
+      1.1968482905982907,
+      0.44337606837606836,
+      1.0512820512820513,
+      1.2852564102564104,
+      1.1696047008547008,
+      1.2793803418803418,
+      0.6597222222222222,
+      0.9850427350427351,
+      0.4732905982905983,
+      0.4895833333333333,
+      0.719017094017094
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7278525641025637,
+      0.7399519230769228,
+      0.7416185897435893,
+      0.7458333333333332,
+      0.7425641025641024
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.679214743589744,
+      0.6924679487179483,
+      0.6926602564102561,
+      0.7026282051282045,
+      0.696442307692308
+    ]
+  }
+}

results_v2/flight_uniform_kmin_57_seed2/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_57_seed2.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.32371794871794873,
+      0.41025641025641024,
+      0.5112179487179487,
+      0.5032051282051282,
+      0.5272435897435898
+    ],
+    "model_accs": [
+      0.32371794871794873,
+      0.41025641025641024,
+      0.5112179487179487,
+      0.5032051282051282,
+      0.5272435897435898
+    ],
+    "model_reward_accs": [
+      0.32371794871794873,
+      0.45032051282051283,
+      0.5160256410256411,
+      0.5496794871794872,
+      0.5753205128205128
+    ],
+    "model_consistencies": [
+      0.3349358974358974,
+      0.6362179487179487,
+      0.6474358974358975,
+      0.6907051282051282,
+      0.6762820512820513
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.0881966756664372,
+      0.9942620813241648,
+      0.9571470977325924,
+      0.9033129773810492,
+      0.8727487988079496
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.4832852564102566,
+      0.5301442307692312,
+      0.5502403846153853,
+      0.5742788461538455,
+      0.5904166666666665
+    ],
+    "model_random_set_idxs": [
+      0.719551282051282,
+      1.2315705128205128,
+      1.2379807692307692,
+      1.0977564102564104,
+      1.0037393162393162,
+      1.1228632478632479,
+      1.171474358974359,
+      0.9853098290598291,
+      1.0945512820512822,
+      1.1274038461538463,
+      0.8314636752136753,
+      1.0819978632478633,
+      1.2091346153846154,
+      0.8221153846153846,
+      1.0253739316239316,
+      1.1511752136752136,
+      0.9169337606837606,
+      0.8643162393162394,
+      0.7096688034188035,
+      0.6757478632478633,
+      1.1207264957264957,
+      1.000534188034188,
+      0.7120726495726496,
+      0.9324252136752137,
+      1.174412393162393,
+      1.0173611111111112,
+      0.9559294871794872,
+      1.1888354700854702,
+      0.7251602564102564,
+      1.0,
+      1.063568376068376,
+      1.0387286324786325,
+      0.9353632478632479,
+      0.9847756410256411,
+      0.9123931623931624,
+      0.8400106837606838,
+      0.5053418803418803,
+      0.9006410256410257,
+      1.1503739316239316,
+      0.9871794871794872,
+      1.1153846153846154,
+      0.9666132478632479,
+      0.7267628205128205,
+      0.8979700854700855,
+      0.9011752136752137,
+      0.6399572649572649,
+      0.7467948717948718,
+      1.3571047008547008,
+      1.4137286324786325,
+      1.1121794871794872,
+      0.9190705128205128,
+      1.189102564102564,
+      1.1981837606837606,
+      1.2457264957264957,
+      0.6623931623931624,
+      1.327190170940171,
+      0.6284722222222222,
+      0.7673611111111112,
+      0.9342948717948718,
+      1.047542735042735,
+      1.0259081196581197,
+      0.7264957264957265,
+      1.0251068376068375,
+      1.0373931623931625,
+      1.3645833333333333,
+      1.0889423076923077,
+      1.0980235042735043,
+      0.6650641025641025,
+      1.0836004273504274,
+      1.0980235042735043,
+      0.7481303418803419,
+      1.0413995726495726,
+      1.1533119658119657,
+      0.8402777777777778,
+      1.1907051282051282,
+      1.077190170940171,
+      1.0721153846153846,
+      1.2000534188034189,
+      1.1038995726495726,
+      1.2278311965811965,
+      0.9871794871794872,
+      0.8055555555555556,
+      1.2604166666666667,
+      1.2248931623931625,
+      0.9356303418803419,
+      1.1610576923076923,
+      0.7270299145299145,
+      0.7128739316239316,
+      1.1009615384615385,
+      1.0392628205128205,
+      0.9166666666666666,
+      1.1394230769230769,
+      1.3709935897435896,
+      1.0125534188034189,
+      0.9826388888888888,
+      1.1236645299145298,
+      0.8611111111111112,
+      0.8886217948717948,
+      0.8052884615384616,
+      0.624465811965812
+    ],
+    "model_probs": [
+      0.0016000654196630375,
+      0.003748241673394777,
+      0.005132361890356189,
+      0.006556662720297068,
+      0.007432930139980243,
+      0.008667737306981557
+    ],
+    "model_info_gains": [
+      0.25418195731636656,
+      0.08460538243226115,
+      0.05622354500962734,
+      0.12903143952005156,
+      0.12857899480743504
+    ],
+    "model_delta_probs": [
+      0.0021481762537317395,
+      0.001384120216961412,
+      0.001424300829940879,
+      0.0008762674196831746,
+      0.0012348071670013146
+    ],
+    "model_expected_reward_accs": [
+      0.32051282051282054,
+      0.47115384615384615,
+      0.5432692307692307,
+      0.5592948717948718,
+      0.5705128205128205
+    ],
+    "model_expected_consistencies": [
+      0.3301282051282051,
+      0.6875,
+      0.7131410256410257,
+      0.7307692307692307,
+      0.7131410256410257
+    ],
+    "model_expected_l2s": [
+      1.356163821748717,
+      1.0556422762641664,
+      0.9586570487983528,
+      0.9051097266727144,
+      0.8599885252430978,
+      0.8279112291583632
+    ],
+    "model_expected_random_set_accs": [
+      0.34895833333333326,
+      0.49910256410256326,
+      0.5466025641025646,
+      0.5755769230769233,
+      0.5956570512820514,
+      0.6110096153846153
+    ],
+    "model_expected_random_set_idxs": [
+      0.7900641025641025,
+      1.279647435897436,
+      1.1848290598290598,
+      1.047275641025641,
+      0.9975961538461539,
+      1.0894764957264957,
+      1.2550747863247864,
+      0.9388354700854701,
+      1.2123397435897436,
+      1.2748397435897436,
+      0.936965811965812,
+      1.0980235042735043,
+      1.155715811965812,
+      0.8645833333333334,
+      1.029647435897436,
+      1.2077991452991452,
+      0.969284188034188,
+      0.8538995726495726,
+      0.6893696581196581,
+      0.7243589743589743,
+      1.1829594017094016,
+      1.0659722222222223,
+      0.7086004273504274,
+      0.9364316239316239,
+      1.1685363247863247,
+      0.9746260683760684,
+      0.9569978632478633,
+      1.2091346153846154,
+      0.7056623931623932,
+      1.014155982905983,
+      1.0584935897435896,
+      1.0008012820512822,
+      0.9799679487179487,
+      1.0504807692307692,
+      0.8811431623931624,
+      0.9206730769230769,
+      0.6225961538461539,
+      0.8563034188034188,
+      1.1931089743589745,
+      1.0002670940170941,
+      1.0961538461538463,
+      0.9580662393162394,
+      0.781784188034188,
+      0.9813034188034188,
+      0.9388354700854701,
+      0.6810897435897436,
+      0.7702991452991453,
+      1.327724358974359,
+      1.3653846153846154,
+      1.0438034188034189,
+      0.9252136752136753,
+      1.1677350427350428,
+      1.1535790598290598,
+      1.1706730769230769,
+      0.6436965811965812,
+      1.2999465811965811,
+      0.7013888888888888,
+      0.7550747863247863,
+      0.906517094017094,
+      1.0595619658119657,
+      0.9893162393162394,
+      1.1055021367521367,
+      1.094818376068376,
+      1.0726495726495726,
+      1.3050213675213675,
+      0.9463141025641025,
+      1.0966880341880343,
+      0.4775641025641026,
+      1.0827991452991452,
+      1.1364850427350428,
+      0.7222222222222222,
+      1.0897435897435896,
+      1.1124465811965811,
+      0.8600427350427351,
+      1.156784188034188,
+      1.048344017094017,
+      1.0638354700854702,
+      1.264155982905983,
+      0.9206730769230769,
+      1.1837606837606838,
+      1.0440705128205128,
+      0.8114316239316239,
+      1.2876602564102564,
+      1.189102564102564,
+      0.8904914529914529,
+      1.0803952991452992,
+      0.7347756410256411,
+      0.7697649572649573,
+      1.0924145299145298,
+      0.9746260683760684,
+      0.7067307692307693,
+      1.1842948717948718,
+      1.359775641025641,
+      0.9879807692307693,
+      0.9508547008547008,
+      1.0101495726495726,
+      0.9180021367521367,
+      0.9337606837606838,
+      0.8902243589743589,
+      0.6135149572649573
+    ],
+    "infer_reward_accs": [
+      0.32532051282051283,
+      0.453525641025641,
+      0.5785256410256411,
+      0.6153846153846154,
+      0.75
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.112815277504166,
+      0.9047236623106071,
+      0.73495870270441,
+      0.610534154103777,
+      0.5394455865522593
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.4796314102564098,
+      0.5834294871794882,
+      0.6624038461538464,
+      0.7162820512820511,
+      0.7489102564102562
+    ],
+    "infer_random_set_idxs": [
+      0.5958867521367521,
+      1.0101495726495726,
+      1.548344017094017,
+      1.1372863247863247,
+      0.9706196581196581,
+      1.34375,
+      1.2419871794871795,
+      0.875534188034188,
+      1.1736111111111112,
+      1.202190170940171,
+      0.5408653846153846,
+      1.419871794871795,
+      1.2446581196581197,
+      0.4810363247863248,
+      0.8298611111111112,
+      1.2099358974358974,
+      1.2104700854700854,
+      0.9602029914529915,
+      0.4970619658119658,
+      0.5531517094017094,
+      1.251068376068376,
+      1.0702457264957266,
+      0.44871794871794873,
+      0.6380876068376068,
+      0.9222756410256411,
+      1.1706730769230769,
+      0.7863247863247863,
+      1.2278311965811965,
+      0.8915598290598291,
+      0.6543803418803419,
+      1.0705128205128205,
+      0.6220619658119658,
+      1.328525641025641,
+      0.6589209401709402,
+      0.9762286324786325,
+      0.5197649572649573,
+      0.38060897435897434,
+      0.5771901709401709,
+      0.8717948717948718,
+      1.0718482905982907,
+      1.2820512820512822,
+      0.7302350427350427,
+      0.6081730769230769,
+      0.562232905982906,
+      0.5737179487179487,
+      0.5178952991452992,
+      0.8838141025641025,
+      1.1610576923076923,
+      1.5280448717948718,
+      1.172809829059829,
+      0.4893162393162393,
+      0.8977029914529915,
+      1.53125,
+      1.1618589743589745,
+      0.6081730769230769,
+      1.3533653846153846,
+      0.4358974358974359,
+      0.7983440170940171,
+      0.7823183760683761,
+      0.9594017094017094,
+      0.8031517094017094,
+      0.655448717948718,
+      1.1135149572649572,
+      0.905448717948718,
+      1.094284188034188,
+      0.9241452991452992,
+      0.8693910256410257,
+      0.7345085470085471,
+      1.2283653846153846,
+      1.3149038461538463,
+      0.6049679487179487,
+      1.1837606837606838,
+      1.4925213675213675,
+      0.8915598290598291,
+      1.4567307692307692,
+      1.2532051282051282,
+      1.1183226495726495,
+      1.3084935897435896,
+      0.9970619658119658,
+      1.0536858974358974,
+      1.2719017094017093,
+      0.6968482905982906,
+      1.4254807692307692,
+      0.9644764957264957,
+      1.046741452991453,
+      1.3149038461538463,
+      0.5921474358974359,
+      0.8480235042735043,
+      1.3790064102564104,
+      0.9289529914529915,
+      0.8811431623931624,
+      1.2299679487179487,
+      1.6442307692307692,
+      0.7911324786324786,
+      0.7238247863247863,
+      1.1786858974358974,
+      0.9228098290598291,
+      1.2102029914529915,
+      0.7788461538461539,
+      0.3888888888888889
+    ],
+    "infer_probs": [
+      0.0016000654196630375,
+      0.004918503210243823,
+      0.01170279399718474,
+      0.022490298223065343,
+      0.03673538918715517,
+      0.05226519630265942
+    ],
+    "infer_info_gains": [
+      1.1741744276081851,
+      0.8859003029359622,
+      0.6689406265138397,
+      0.5051128296285503,
+      0.37411798569848426
+    ],
+    "infer_delta_probs": [
+      0.003318437790580785,
+      0.006784290786940918,
+      0.010787504225880604,
+      0.014245090964089822,
+      0.01552980711550425
+    ],
+    "infer_expected_reward_accs": [
+      0.32051282051282054,
+      0.5160256410256411,
+      0.7435897435897436,
+      0.9006410256410257,
+      0.9615384615384616
+    ],
+    "infer_expected_l2s": [
+      1.356163821748717,
+      0.9223112584703202,
+      0.7149921960588865,
+      0.5854926639304799,
+      0.5056043881368028,
+      0.45441234426731686
+    ],
+    "infer_expected_random_set_accs": [
+      0.34895833333333326,
+      0.5688782051282042,
+      0.6676121794871798,
+      0.7280448717948711,
+      0.7644711538461544,
+      0.7889423076923069
+    ],
+    "infer_expected_random_set_idxs": [
+      0.7486645299145299,
+      1.0694444444444444,
+      1.4607371794871795,
+      1.0918803418803418,
+      0.9735576923076923,
+      1.2262286324786325,
+      1.2262286324786325,
+      0.8036858974358975,
+      1.2425213675213675,
+      1.2694978632478633,
+      0.7446581196581197,
+      1.2262286324786325,
+      1.1883012820512822,
+      0.6808226495726496,
+      0.9764957264957265,
+      1.1971153846153846,
+      1.0742521367521367,
+      0.9767628205128205,
+      0.530715811965812,
+      0.6367521367521367,
+      1.1364850427350428,
+      1.0048076923076923,
+      0.5141559829059829,
+      0.7516025641025641,
+      0.9409722222222222,
+      1.0534188034188035,
+      0.8189102564102564,
+      1.1872329059829059,
+      0.8709935897435898,
+      0.7689636752136753,
+      1.1610576923076923,
+      0.7564102564102564,
+      1.1343482905982907,
+      0.7938034188034188,
+      0.9503205128205128,
+      0.7775106837606838,
+      0.6765491452991453,
+      0.6535790598290598,
+      0.9805021367521367,
+      1.0371260683760684,
+      1.1268696581196582,
+      0.7390491452991453,
+      0.6637286324786325,
+      0.8095619658119658,
+      0.6762820512820513,
+      0.6680021367521367,
+      0.9185363247863247,
+      1.123931623931624,
+      1.4425747863247864,
+      1.1022970085470085,
+      0.6621260683760684,
+      0.9676816239316239,
+      1.4014423076923077,
+      1.201655982905983,
+      0.6808226495726496,
+      1.2179487179487178,
+      0.5595619658119658,
+      0.8210470085470085,
+      0.8309294871794872,
+      0.9930555555555556,
+      0.8464209401709402,
+      1.1311431623931625,
+      1.0966880341880343,
+      0.9511217948717948,
+      1.1274038461538463,
+      0.8221153846153846,
+      0.9471153846153846,
+      0.594017094017094,
+      1.268162393162393,
+      1.2684294871794872,
+      0.7334401709401709,
+      1.0633012820512822,
+      1.3792735042735043,
+      0.9602029914529915,
+      1.2887286324786325,
+      1.1351495726495726,
+      1.0555555555555556,
+      1.2529380341880343,
+      0.813034188034188,
+      1.0034722222222223,
+      1.2139423076923077,
+      0.6749465811965812,
+      1.3349358974358974,
+      1.061431623931624,
+      1.1055021367521367,
+      1.1517094017094016,
+      0.6436965811965812,
+      0.9078525641025641,
+      1.2860576923076923,
+      0.8613782051282052,
+      0.6412927350427351,
+      1.2227564102564104,
+      1.4919871794871795,
+      0.8042200854700855,
+      0.8319978632478633,
+      1.078258547008547,
+      0.9949252136752137,
+      1.123931623931624,
+      0.7636217948717948,
+      0.5339209401709402
+    ],
+    "kls": [
+      0.0,
+      1.492311314625048,
+      2.276640642019102,
+      2.811705881409954,
+      3.125403336639521,
+      3.356271191750817
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.46690705128205084,
+      0.4962179487179485,
+      0.5199519230769232,
+      0.54125,
+      0.555432692307693
+    ],
+    "model_random_set_direct_idxs": [
+      1.126068376068376,
+      1.1680021367521367,
+      1.3683226495726495,
+      1.3469551282051282,
+      1.0392628205128205,
+      0.6388888888888888,
+      0.7962072649572649,
+      0.7339743589743589,
+      1.0659722222222223,
+      1.513621794871795,
+      1.3122329059829059,
+      0.8421474358974359,
+      1.3557692307692308,
+      0.8798076923076923,
+      1.422542735042735,
+      1.4730235042735043,
+      1.2230235042735043,
+      1.0,
+      0.7345085470085471,
+      0.9009081196581197,
+      0.6690705128205128,
+      1.1650641025641026,
+      0.8239850427350427,
+      1.2668269230769231,
+      0.9594017094017094,
+      0.8731303418803419,
+      1.0438034188034189,
+      1.3691239316239316,
+      0.7235576923076923,
+      1.2481303418803418,
+      1.3947649572649572,
+      1.296207264957265,
+      0.46474358974358976,
+      1.3231837606837606,
+      0.8528311965811965,
+      1.110042735042735,
+      0.8095619658119658,
+      0.8007478632478633,
+      1.0798611111111112,
+      0.7206196581196581,
+      1.2238247863247864,
+      1.2644230769230769,
+      0.7775106837606838,
+      1.1228632478632479,
+      1.0860042735042734,
+      0.6394230769230769,
+      0.8530982905982906,
+      1.3693910256410255,
+      1.2580128205128205,
+      1.0331196581196582,
+      1.1802884615384615,
+      0.8498931623931624,
+      1.2043269230769231,
+      1.2991452991452992,
+      1.0825320512820513,
+      1.2694978632478633,
+      0.8389423076923077,
+      0.7377136752136753,
+      1.2745726495726495,
+      0.9495192307692307,
+      1.0651709401709402,
+      0.7622863247863247,
+      1.1688034188034189,
+      0.6461004273504274,
+      1.2938034188034189,
+      0.905715811965812,
+      0.9877136752136753,
+      0.4361645299145299,
+      0.8245192307692307,
+      1.1188568376068375,
+      1.0405982905982907,
+      0.7286324786324786,
+      1.2540064102564104,
+      0.8306623931623932,
+      1.2003205128205128,
+      0.7644230769230769,
+      0.7580128205128205,
+      1.0416666666666667,
+      0.8685897435897436,
+      1.2110042735042734,
+      1.1180555555555556,
+      0.8036858974358975,
+      1.0689102564102564,
+      0.9874465811965812,
+      1.1963141025641026,
+      1.1209935897435896,
+      0.6858974358974359,
+      0.7670940170940171,
+      1.0275106837606838,
+      1.2331730769230769,
+      0.5093482905982906,
+      1.049412393162393,
+      1.3293269230769231,
+      1.1645299145299146,
+      1.2884615384615385,
+      0.6808226495726496,
+      0.9829059829059829,
+      0.4497863247863248,
+      0.5216346153846154,
+      0.7174145299145299
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7287339743589738,
+      0.730112179487179,
+      0.7364903846153856,
+      0.734054487179487,
+      0.7404166666666665
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.6789583333333334,
+      0.6794391025641029,
+      0.6873557692307695,
+      0.6933333333333332,
+      0.692692307692308
+    ]
+  }
+}

results_v2/flight_uniform_kmin_74_seed1/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_74_seed1.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.11217948717948718,
+      0.33814102564102566,
+      0.40705128205128205,
+      0.39903846153846156,
+      0.44711538461538464
+    ],
+    "model_accs": [
+      0.11217948717948718,
+      0.33814102564102566,
+      0.40705128205128205,
+      0.39903846153846156,
+      0.44711538461538464
+    ],
+    "model_reward_accs": [
+      0.3125,
+      0.34134615384615385,
+      0.42467948717948717,
+      0.4567307692307692,
+      0.47115384615384615
+    ],
+    "model_consistencies": [
+      0.34134615384615385,
+      0.6666666666666666,
+      0.6538461538461539,
+      0.6442307692307693,
+      0.6842948717948718
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.1084234439730003,
+      1.0422582889356224,
+      1.0096403401715524,
+      0.9560923563325124,
+      0.9522267056618691
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.4743429487179491,
+      0.5085897435897438,
+      0.5259294871794874,
+      0.548189102564103,
+      0.5537660256410261
+    ],
+    "model_random_set_idxs": [
+      0.6808226495726496,
+      1.2361111111111112,
+      1.2446581196581197,
+      1.0902777777777777,
+      0.9874465811965812,
+      1.1201923076923077,
+      1.186431623931624,
+      0.9676816239316239,
+      1.1105769230769231,
+      1.138621794871795,
+      0.8181089743589743,
+      1.0884081196581197,
+      1.2008547008547008,
+      0.8047542735042735,
+      1.0173611111111112,
+      1.1674679487179487,
+      0.9244123931623932,
+      0.8707264957264957,
+      0.7176816239316239,
+      0.6776175213675214,
+      1.1356837606837606,
+      0.9898504273504274,
+      0.7094017094017094,
+      0.9185363247863247,
+      1.1765491452991452,
+      1.0285790598290598,
+      0.9663461538461539,
+      1.2115384615384615,
+      0.7136752136752137,
+      0.9943910256410257,
+      1.0587606837606838,
+      1.0264423076923077,
+      0.9532585470085471,
+      0.9850427350427351,
+      0.9321581196581197,
+      0.8226495726495726,
+      0.47489316239316237,
+      0.8816773504273504,
+      1.1762820512820513,
+      0.9986645299145299,
+      1.1565170940170941,
+      0.9561965811965812,
+      0.7270299145299145,
+      0.8808760683760684,
+      0.9030448717948718,
+      0.6100427350427351,
+      0.7339743589743589,
+      1.3747329059829059,
+      1.422275641025641,
+      1.1180555555555556,
+      0.9003739316239316,
+      1.1845619658119657,
+      1.2118055555555556,
+      1.2291666666666667,
+      0.6298076923076923,
+      1.342147435897436,
+      0.6236645299145299,
+      0.7668269230769231,
+      0.9308226495726496,
+      1.0491452991452992,
+      1.0122863247863247,
+      0.7112713675213675,
+      1.0416666666666667,
+      1.032318376068376,
+      1.3517628205128205,
+      1.095352564102564,
+      1.092681623931624,
+      0.6605235042735043,
+      1.1164529914529915,
+      1.1185897435897436,
+      0.719551282051282,
+      1.0419337606837606,
+      1.1461004273504274,
+      0.8514957264957265,
+      1.2003205128205128,
+      1.080128205128205,
+      1.095352564102564,
+      1.1992521367521367,
+      1.108707264957265,
+      1.2387820512820513,
+      0.9826388888888888,
+      0.8052884615384616,
+      1.2601495726495726,
+      1.2184829059829059,
+      0.936965811965812,
+      1.1485042735042734,
+      0.7094017094017094,
+      0.7216880341880342,
+      1.0950854700854702,
+      1.0397970085470085,
+      0.9233440170940171,
+      1.155181623931624,
+      1.3840811965811965,
+      1.0144230769230769,
+      0.9658119658119658,
+      1.1247329059829059,
+      0.8688568376068376,
+      0.8859508547008547,
+      0.8151709401709402,
+      0.6354166666666666
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.003220695232451831,
+      0.0036661737750604538,
+      0.003995221436018899,
+      0.004813175252951833,
+      0.005429075906942735
+    ],
+    "model_info_gains": [
+      0.4082279904882643,
+      -0.04044895948869193,
+      -0.03763803184863857,
+      0.15134285377903828,
+      -0.034448474385289946
+    ],
+    "model_delta_probs": [
+      0.0016206427334232477,
+      0.00044547854260862244,
+      0.0003290476609584458,
+      0.0008179538169329339,
+      0.000615900653990901
+    ],
+    "model_expected_reward_accs": [
+      0.3189102564102564,
+      0.33974358974358976,
+      0.41025641025641024,
+      0.4423076923076923,
+      0.4807692307692308
+    ],
+    "model_expected_consistencies": [
+      0.34455128205128205,
+      0.7516025641025641,
+      0.7211538461538461,
+      0.7083333333333334,
+      0.7323717948717948
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.0744161990193597,
+      1.0104384249117067,
+      0.9726364427216643,
+      0.9226112527053175,
+      0.9000387931743277
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.49025641025641026,
+      0.524342948717949,
+      0.5421314102564102,
+      0.5635897435897435,
+      0.5754487179487183
+    ],
+    "model_expected_random_set_idxs": [
+      0.9364316239316239,
+      1.282852564102564,
+      1.2126068376068375,
+      1.0413995726495726,
+      0.9986645299145299,
+      1.108707264957265,
+      1.248931623931624,
+      0.9153311965811965,
+      1.2280982905982907,
+      1.2793803418803418,
+      0.9420405982905983,
+      1.1124465811965811,
+      1.1444978632478633,
+      0.8381410256410257,
+      1.0138888888888888,
+      1.2184829059829059,
+      0.9754273504273504,
+      0.8565705128205128,
+      0.7061965811965812,
+      0.7152777777777778,
+      1.1960470085470085,
+      1.0357905982905984,
+      0.6957799145299145,
+      0.9073183760683761,
+      1.1575854700854702,
+      0.9756944444444444,
+      0.9786324786324786,
+      1.2168803418803418,
+      0.7005876068376068,
+      1.0109508547008548,
+      1.0603632478632479,
+      0.9700854700854701,
+      0.9869123931623932,
+      1.0435363247863247,
+      0.8904914529914529,
+      0.9169337606837606,
+      0.6153846153846154,
+      0.8327991452991453,
+      1.1907051282051282,
+      1.016025641025641,
+      1.1124465811965811,
+      0.9431089743589743,
+      0.7852564102564102,
+      0.9813034188034188,
+      0.9393696581196581,
+      0.6543803418803419,
+      0.7524038461538461,
+      1.3269230769230769,
+      1.3669871794871795,
+      1.047008547008547,
+      0.9110576923076923,
+      1.1541132478632479,
+      1.1591880341880343,
+      1.1746794871794872,
+      0.6185897435897436,
+      1.3146367521367521,
+      0.6979166666666666,
+      0.7732371794871795,
+      0.9153311965811965,
+      1.0673076923076923,
+      0.9644764957264957,
+      1.0961538461538463,
+      1.1111111111111112,
+      1.0715811965811965,
+      1.2916666666666667,
+      0.9495192307692307,
+      1.0836004273504274,
+      0.46287393162393164,
+      1.1044337606837606,
+      1.1525106837606838,
+      0.6912393162393162,
+      1.0889423076923077,
+      1.1028311965811965,
+      0.8683226495726496,
+      1.172275641025641,
+      1.0587606837606838,
+      1.0822649572649572,
+      1.2604166666666667,
+      0.9137286324786325,
+      1.1861645299145298,
+      1.0560897435897436,
+      0.8111645299145299,
+      1.2908653846153846,
+      1.1971153846153846,
+      0.8950320512820513,
+      1.077991452991453,
+      0.7077991452991453,
+      0.7841880341880342,
+      1.1041666666666667,
+      0.9620726495726496,
+      0.686965811965812,
+      1.2013888888888888,
+      1.3576388888888888,
+      0.9719551282051282,
+      0.9316239316239316,
+      1.0064102564102564,
+      0.9516559829059829,
+      0.9644764957264957,
+      0.8963675213675214,
+      0.6212606837606838
+    ],
+    "infer_reward_accs": [
+      0.3173076923076923,
+      0.3317307692307692,
+      0.44551282051282054,
+      0.5448717948717948,
+      0.6987179487179487
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.1106956049223664,
+      0.8220930587233545,
+      0.6031831106344999,
+      0.4384826838573109,
+      0.3442626463431043
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.47785256410256505,
+      0.6191025641025646,
+      0.7204166666666669,
+      0.7945993589743594,
+      0.834294871794872
+    ],
+    "infer_random_set_idxs": [
+      0.6840277777777778,
+      1.0184294871794872,
+      1.5202991452991452,
+      1.1135149572649572,
+      0.9668803418803419,
+      1.3004807692307692,
+      1.2160790598290598,
+      0.8514957264957265,
+      1.1674679487179487,
+      1.2110042735042734,
+      0.6311431623931624,
+      1.3392094017094016,
+      1.2390491452991452,
+      0.530715811965812,
+      0.8950320512820513,
+      1.2096688034188035,
+      1.1808226495726495,
+      0.9986645299145299,
+      0.5093482905982906,
+      0.6044337606837606,
+      1.1920405982905984,
+      1.0333867521367521,
+      0.4861111111111111,
+      0.7080662393162394,
+      0.9126602564102564,
+      1.1639957264957266,
+      0.8095619658119658,
+      1.2123397435897436,
+      0.8952991452991453,
+      0.6928418803418803,
+      1.110844017094017,
+      0.6677350427350427,
+      1.2430555555555556,
+      0.6899038461538461,
+      0.9845085470085471,
+      0.5456730769230769,
+      0.4797008547008547,
+      0.5953525641025641,
+      0.8864850427350427,
+      1.044337606837607,
+      1.2222222222222223,
+      0.7510683760683761,
+      0.6591880341880342,
+      0.6225961538461539,
+      0.5900106837606838,
+      0.6095085470085471,
+      0.8995726495726496,
+      1.107905982905983,
+      1.467681623931624,
+      1.1274038461538463,
+      0.5496794871794872,
+      0.8961004273504274,
+      1.4877136752136753,
+      1.1955128205128205,
+      0.6642628205128205,
+      1.294337606837607,
+      0.46875,
+      0.8303952991452992,
+      0.8474893162393162,
+      0.9839743589743589,
+      0.8111645299145299,
+      0.7069978632478633,
+      1.0974893162393162,
+      0.8827457264957265,
+      1.1161858974358974,
+      0.9049145299145299,
+      0.8910256410256411,
+      0.7267628205128205,
+      1.2248931623931625,
+      1.2735042735042734,
+      0.6754807692307693,
+      1.1159188034188035,
+      1.4660790598290598,
+      0.9364316239316239,
+      1.390491452991453,
+      1.2086004273504274,
+      1.077991452991453,
+      1.2302350427350428,
+      1.0013354700854702,
+      1.0133547008547008,
+      1.2216880341880343,
+      0.7080662393162394,
+      1.3544337606837606,
+      0.9794337606837606,
+      1.0873397435897436,
+      1.2849893162393162,
+      0.6733440170940171,
+      0.8867521367521367,
+      1.3365384615384615,
+      0.9537927350427351,
+      0.8627136752136753,
+      1.1760149572649572,
+      1.5830662393162394,
+      0.8413461538461539,
+      0.8055555555555556,
+      1.139957264957265,
+      0.9489850427350427,
+      1.139957264957265,
+      0.7649572649572649,
+      0.4545940170940171
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.005251737105755894,
+      0.016062439503888852,
+      0.04060428231376789,
+      0.08074720405495146,
+      0.13537531990088889
+    ],
+    "infer_info_gains": [
+      1.258480140833615,
+      1.1579873356682964,
+      0.9140977199681092,
+      0.697910642498664,
+      0.5123241245564005
+    ],
+    "infer_delta_probs": [
+      0.0036516846067273104,
+      0.010810702398132959,
+      0.024541842809879037,
+      0.04014292174118358,
+      0.05462811584593743
+    ],
+    "infer_expected_reward_accs": [
+      0.3189102564102564,
+      0.23557692307692307,
+      0.4855769230769231,
+      0.6858974358974359,
+      0.8669871794871795
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      0.904900242287776,
+      0.6351674276610744,
+      0.47184528684682814,
+      0.3572292371081554,
+      0.29874005170694395
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5754967948717944,
+      0.7053525641025633,
+      0.7805288461538458,
+      0.8309134615384619,
+      0.8593429487179493
+    ],
+    "infer_expected_random_set_idxs": [
+      0.9209401709401709,
+      1.0747863247863247,
+      1.4508547008547008,
+      1.0683760683760684,
+      0.9577991452991453,
+      1.2110042735042734,
+      1.2136752136752136,
+      0.7967414529914529,
+      1.2601495726495726,
+      1.268162393162393,
+      0.7844551282051282,
+      1.1947115384615385,
+      1.1813568376068375,
+      0.688034188034188,
+      0.9850427350427351,
+      1.2019230769230769,
+      1.0622329059829059,
+      1.0253739316239316,
+      0.5470085470085471,
+      0.6834935897435898,
+      1.1258012820512822,
+      0.9631410256410257,
+      0.5432692307692307,
+      0.7895299145299145,
+      0.9521901709401709,
+      1.0518162393162394,
+      0.8450854700854701,
+      1.1931089743589745,
+      0.8888888888888888,
+      0.7670940170940171,
+      1.1674679487179487,
+      0.7783119658119658,
+      1.1041666666666667,
+      0.7873931623931624,
+      0.9644764957264957,
+      0.7262286324786325,
+      0.6920405982905983,
+      0.6450320512820513,
+      0.9850427350427351,
+      1.018162393162393,
+      1.0993589743589745,
+      0.7593482905982906,
+      0.6824252136752137,
+      0.7825854700854701,
+      0.6674679487179487,
+      0.687767094017094,
+      0.9174679487179487,
+      1.0884081196581197,
+      1.3982371794871795,
+      1.0769230769230769,
+      0.6543803418803419,
+      0.9580662393162394,
+      1.3934294871794872,
+      1.2267628205128205,
+      0.6899038461538461,
+      1.1915064102564104,
+      0.5600961538461539,
+      0.8352029914529915,
+      0.8386752136752137,
+      1.0341880341880343,
+      0.8458867521367521,
+      1.1121794871794872,
+      1.1017628205128205,
+      0.9142628205128205,
+      1.1354166666666667,
+      0.8314636752136753,
+      0.9548611111111112,
+      0.5988247863247863,
+      1.2855235042735043,
+      1.2697649572649572,
+      0.7550747863247863,
+      1.0392628205128205,
+      1.360042735042735,
+      0.9983974358974359,
+      1.2743055555555556,
+      1.1322115384615385,
+      1.0574252136752136,
+      1.2166132478632479,
+      0.8349358974358975,
+      0.9877136752136753,
+      1.1770833333333333,
+      0.6845619658119658,
+      1.2772435897435896,
+      1.0822649572649572,
+      1.107371794871795,
+      1.1442307692307692,
+      0.6752136752136753,
+      0.9340277777777778,
+      1.265491452991453,
+      0.8327991452991453,
+      0.6541132478632479,
+      1.1952457264957266,
+      1.4660790598290598,
+      0.8402777777777778,
+      0.843482905982906,
+      1.0582264957264957,
+      1.0211004273504274,
+      1.095352564102564,
+      0.7606837606837606,
+      0.5499465811965812
+    ],
+    "kls": [
+      0.0,
+      1.553573343571621,
+      2.7237101585631667,
+      3.539764044370576,
+      4.048182513896174,
+      4.440994734802607
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.452195512820513,
+      0.4704326923076916,
+      0.4890705128205127,
+      0.5030769230769228,
+      0.5201602564102568
+    ],
+    "model_random_set_direct_idxs": [
+      1.1172542735042734,
+      1.172008547008547,
+      1.4265491452991452,
+      1.3183760683760684,
+      1.014155982905983,
+      0.6469017094017094,
+      0.7905982905982906,
+      0.7248931623931624,
+      1.0878739316239316,
+      1.514957264957265,
+      1.3036858974358974,
+      0.8469551282051282,
+      1.3378739316239316,
+      0.8875534188034188,
+      1.4260149572649572,
+      1.4770299145299146,
+      1.2064636752136753,
+      1.032852564102564,
+      0.7459935897435898,
+      0.8993055555555556,
+      0.6530448717948718,
+      1.1316773504273505,
+      0.8162393162393162,
+      1.265224358974359,
+      0.9516559829059829,
+      0.8811431623931624,
+      1.0729166666666667,
+      1.3541666666666667,
+      0.7230235042735043,
+      1.236912393162393,
+      1.4043803418803418,
+      1.2462606837606838,
+      0.4722222222222222,
+      1.3015491452991452,
+      0.8806089743589743,
+      1.0667735042735043,
+      0.8186431623931624,
+      0.7700320512820513,
+      1.0440705128205128,
+      0.6690705128205128,
+      1.2299679487179487,
+      1.2532051282051282,
+      0.7791132478632479,
+      1.1140491452991452,
+      1.0544871794871795,
+      0.5913461538461539,
+      0.8279914529914529,
+      1.3512286324786325,
+      1.2737713675213675,
+      1.0192307692307692,
+      1.1471688034188035,
+      0.8384081196581197,
+      1.2299679487179487,
+      1.3052884615384615,
+      1.0683760683760684,
+      1.2564102564102564,
+      0.8239850427350427,
+      0.7521367521367521,
+      1.2855235042735043,
+      0.9607371794871795,
+      1.0411324786324787,
+      0.7727029914529915,
+      1.172542735042735,
+      0.6183226495726496,
+      1.2913995726495726,
+      0.9102564102564102,
+      0.9847756410256411,
+      0.42094017094017094,
+      0.8555021367521367,
+      1.1434294871794872,
+      1.0037393162393162,
+      0.7072649572649573,
+      1.2903311965811965,
+      0.8322649572649573,
+      1.1808226495726495,
+      0.7638888888888888,
+      0.7596153846153846,
+      1.0349893162393162,
+      0.8482905982905983,
+      1.1866987179487178,
+      1.1193910256410255,
+      0.8050213675213675,
+      1.0288461538461537,
+      1.0074786324786325,
+      1.2067307692307692,
+      1.0969551282051282,
+      0.6720085470085471,
+      0.7847222222222222,
+      1.061965811965812,
+      1.2083333333333333,
+      0.5032051282051282,
+      1.046207264957265,
+      1.3501602564102564,
+      1.1794871794871795,
+      1.3007478632478633,
+      0.6586538461538461,
+      0.9810363247863247,
+      0.46073717948717946,
+      0.5010683760683761,
+      0.7248931623931624
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7187500000000006,
+      0.7236057692307687,
+      0.7316185897435898,
+      0.7262339743589744,
+      0.7303205128205134
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.6640544871794873,
+      0.6733493589743589,
+      0.6796634615384612,
+      0.6739262820512826,
+      0.6842147435897431
+    ]
+  }
+}

results_v2/flight_uniform_kmin_76_seed2/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_76_seed2.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.08493589743589744,
+      0.3189102564102564,
+      0.3717948717948718,
+      0.4150641025641026,
+      0.42788461538461536
+    ],
+    "model_accs": [
+      0.08493589743589744,
+      0.3189102564102564,
+      0.3717948717948718,
+      0.4150641025641026,
+      0.42788461538461536
+    ],
+    "model_reward_accs": [
+      0.3141025641025641,
+      0.3108974358974359,
+      0.38782051282051283,
+      0.453525641025641,
+      0.48717948717948717
+    ],
+    "model_consistencies": [
+      0.3573717948717949,
+      0.6618589743589743,
+      0.6746794871794872,
+      0.7035256410256411,
+      0.6650641025641025
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.1420957565705852,
+      1.0384693886409238,
+      0.9992888135605793,
+      0.9454094820036215,
+      0.9339735567594766
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.456778846153846,
+      0.507451923076923,
+      0.5281570512820508,
+      0.5527403846153852,
+      0.5574038461538464
+    ],
+    "model_random_set_idxs": [
+      0.7259615384615384,
+      1.2219551282051282,
+      1.248397435897436,
+      1.1159188034188035,
+      1.0104166666666667,
+      1.1188568376068375,
+      1.169871794871795,
+      0.9981303418803419,
+      1.09375,
+      1.126602564102564,
+      0.8199786324786325,
+      1.0715811965811965,
+      1.2227564102564104,
+      0.7994123931623932,
+      1.0192307692307692,
+      1.15625,
+      0.9407051282051282,
+      0.843482905982906,
+      0.7155448717948718,
+      0.6885683760683761,
+      1.1028311965811965,
+      1.016559829059829,
+      0.7166132478632479,
+      0.9073183760683761,
+      1.169337606837607,
+      1.0008012820512822,
+      0.9642094017094017,
+      1.1931089743589745,
+      0.7206196581196581,
+      1.0202991452991452,
+      1.0595619658119657,
+      1.047542735042735,
+      0.9401709401709402,
+      1.0120192307692308,
+      0.8982371794871795,
+      0.8619123931623932,
+      0.4863782051282051,
+      0.9214743589743589,
+      1.1565170940170941,
+      1.0114850427350428,
+      1.138087606837607,
+      0.9663461538461539,
+      0.7329059829059829,
+      0.9139957264957265,
+      0.9011752136752137,
+      0.6386217948717948,
+      0.7358440170940171,
+      1.3493589743589745,
+      1.4038461538461537,
+      1.1306089743589745,
+      0.9292200854700855,
+      1.1773504273504274,
+      1.2008547008547008,
+      1.2302350427350428,
+      0.6802884615384616,
+      1.3327991452991452,
+      0.6551816239316239,
+      0.7694978632478633,
+      0.9401709401709402,
+      1.0400641025641026,
+      1.0381944444444444,
+      0.717948717948718,
+      1.0288461538461537,
+      1.0440705128205128,
+      1.357905982905983,
+      1.077190170940171,
+      1.0956196581196582,
+      0.6615918803418803,
+      1.0576923076923077,
+      1.0825320512820513,
+      0.749732905982906,
+      1.0355235042735043,
+      1.1322115384615385,
+      0.8167735042735043,
+      1.205128205128205,
+      1.078792735042735,
+      1.063034188034188,
+      1.203792735042735,
+      1.1127136752136753,
+      1.235042735042735,
+      0.9970619658119658,
+      0.8042200854700855,
+      1.2446581196581197,
+      1.2094017094017093,
+      0.937232905982906,
+      1.141025641025641,
+      0.7457264957264957,
+      0.7112713675213675,
+      1.0964209401709402,
+      1.0526175213675213,
+      0.9086538461538461,
+      1.1346153846153846,
+      1.373397435897436,
+      1.0224358974358974,
+      0.9797008547008547,
+      1.1439636752136753,
+      0.843482905982906,
+      0.8830128205128205,
+      0.8050213675213675,
+      0.6225961538461539
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.0026384032732234795,
+      0.003816768010953462,
+      0.004225406430476342,
+      0.005054352651727028,
+      0.005362860204818331
+    ],
+    "model_info_gains": [
+      0.3765875878454145,
+      0.039868019043888314,
+      0.07560391561047239,
+      0.04083944542713509,
+      0.07489190538584009
+    ],
+    "model_delta_probs": [
+      0.001038350774194896,
+      0.0011783647377299827,
+      0.0004086384195228805,
+      0.0008289462212506857,
+      0.0003085075530913027
+    ],
+    "model_expected_reward_accs": [
+      0.3157051282051282,
+      0.30448717948717946,
+      0.40384615384615385,
+      0.4567307692307692,
+      0.4855769230769231
+    ],
+    "model_expected_consistencies": [
+      0.36217948717948717,
+      0.7227564102564102,
+      0.6939102564102564,
+      0.7564102564102564,
+      0.7259615384615384
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.088603525762798,
+      0.997415985094544,
+      0.9576071190031552,
+      0.9179038899548285,
+      0.8809692194894883
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.4824519230769231,
+      0.5289903846153848,
+      0.5473397435897441,
+      0.5668910256410259,
+      0.5825641025641041
+    ],
+    "model_expected_random_set_idxs": [
+      0.9895833333333334,
+      1.2732371794871795,
+      1.201655982905983,
+      1.0649038461538463,
+      1.0365918803418803,
+      1.0956196581196582,
+      1.2393162393162394,
+      0.9572649572649573,
+      1.2086004273504274,
+      1.2889957264957266,
+      0.9489850427350427,
+      1.0876068376068375,
+      1.1752136752136753,
+      0.8621794871794872,
+      1.0109508547008548,
+      1.202724358974359,
+      0.9970619658119658,
+      0.8336004273504274,
+      0.7147435897435898,
+      0.7345085470085471,
+      1.173878205128205,
+      1.063034188034188,
+      0.7315705128205128,
+      0.8926282051282052,
+      1.1517094017094016,
+      0.9556623931623932,
+      0.9783653846153846,
+      1.1810897435897436,
+      0.7102029914529915,
+      1.0237713675213675,
+      1.045405982905983,
+      0.9906517094017094,
+      0.9802350427350427,
+      1.093215811965812,
+      0.8592414529914529,
+      0.9393696581196581,
+      0.6193910256410257,
+      0.8693910256410257,
+      1.1826923076923077,
+      1.0320512820512822,
+      1.1063034188034189,
+      0.9652777777777778,
+      0.8135683760683761,
+      0.9959935897435898,
+      0.9465811965811965,
+      0.6928418803418803,
+      0.7446581196581197,
+      1.327457264957265,
+      1.3424145299145298,
+      1.0360576923076923,
+      0.9436431623931624,
+      1.1461004273504274,
+      1.1490384615384615,
+      1.1514423076923077,
+      0.6522435897435898,
+      1.3146367521367521,
+      0.7403846153846154,
+      0.7721688034188035,
+      0.9046474358974359,
+      1.0496794871794872,
+      0.9927884615384616,
+      1.1207264957264957,
+      1.1038995726495726,
+      1.1060363247863247,
+      1.2857905982905984,
+      0.9399038461538461,
+      1.0849358974358974,
+      0.4671474358974359,
+      1.0584935897435896,
+      1.1303418803418803,
+      0.7342414529914529,
+      1.0902777777777777,
+      1.079059829059829,
+      0.8450854700854701,
+      1.1933760683760684,
+      1.0368589743589745,
+      1.0555555555555556,
+      1.2740384615384615,
+      0.9134615384615384,
+      1.1955128205128205,
+      1.0739850427350428,
+      0.8135683760683761,
+      1.2852564102564104,
+      1.1826923076923077,
+      0.8720619658119658,
+      1.076121794871795,
+      0.7417200854700855,
+      0.7761752136752137,
+      1.092681623931624,
+      0.9586004273504274,
+      0.6744123931623932,
+      1.200587606837607,
+      1.344284188034188,
+      0.9885149572649573,
+      0.9465811965811965,
+      1.0173611111111112,
+      0.9313568376068376,
+      0.9254807692307693,
+      0.8982371794871795,
+      0.6129807692307693
+    ],
+    "infer_reward_accs": [
+      0.3157051282051282,
+      0.3541666666666667,
+      0.4391025641025641,
+      0.5689102564102564,
+      0.6698717948717948
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.1017450714231103,
+      0.8159186098004803,
+      0.5815104939767978,
+      0.42963865697032944,
+      0.3310353286494801
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.4877884615384615,
+      0.6251282051282049,
+      0.7315064102564113,
+      0.7974358974358982,
+      0.8408493589743596
+    ],
+    "infer_random_set_idxs": [
+      0.6746794871794872,
+      1.030181623931624,
+      1.5237713675213675,
+      1.1209935897435896,
+      0.9903846153846154,
+      1.2844551282051282,
+      1.2110042735042734,
+      0.8453525641025641,
+      1.1778846153846154,
+      1.2115384615384615,
+      0.6183226495726496,
+      1.3215811965811965,
+      1.235309829059829,
+      0.5408653846153846,
+      0.8808760683760684,
+      1.1952457264957266,
+      1.1535790598290598,
+      0.9949252136752137,
+      0.5168269230769231,
+      0.6092414529914529,
+      1.1984508547008548,
+      1.0379273504273505,
+      0.4935897435897436,
+      0.6661324786324786,
+      0.9209401709401709,
+      1.1503739316239316,
+      0.8247863247863247,
+      1.2136752136752136,
+      0.8846153846153846,
+      0.7059294871794872,
+      1.109508547008547,
+      0.6784188034188035,
+      1.2470619658119657,
+      0.717948717948718,
+      0.9933226495726496,
+      0.5643696581196581,
+      0.4639423076923077,
+      0.5956196581196581,
+      0.8912927350427351,
+      1.0697115384615385,
+      1.2371794871794872,
+      0.7724358974358975,
+      0.6428952991452992,
+      0.625267094017094,
+      0.6151175213675214,
+      0.5876068376068376,
+      0.8608440170940171,
+      1.1140491452991452,
+      1.483440170940171,
+      1.1388888888888888,
+      0.5523504273504274,
+      0.9030448717948718,
+      1.4719551282051282,
+      1.1850961538461537,
+      0.6466346153846154,
+      1.297008547008547,
+      0.5157585470085471,
+      0.8416132478632479,
+      0.8464209401709402,
+      1.0029380341880343,
+      0.7916666666666666,
+      0.7013888888888888,
+      1.1153846153846154,
+      0.9051816239316239,
+      1.1287393162393162,
+      0.9249465811965812,
+      0.9134615384615384,
+      0.7288995726495726,
+      1.2195512820512822,
+      1.2617521367521367,
+      0.6773504273504274,
+      1.1428952991452992,
+      1.455128205128205,
+      0.9241452991452992,
+      1.3958333333333333,
+      1.2075320512820513,
+      1.0691773504273505,
+      1.2502670940170941,
+      0.9946581196581197,
+      1.018162393162393,
+      1.263621794871795,
+      0.7198183760683761,
+      1.3576388888888888,
+      0.9797008547008547,
+      1.0659722222222223,
+      1.267628205128205,
+      0.6423611111111112,
+      0.8931623931623932,
+      1.360309829059829,
+      0.9433760683760684,
+      0.8330662393162394,
+      1.2061965811965811,
+      1.5739850427350428,
+      0.8346688034188035,
+      0.7986111111111112,
+      1.1517094017094016,
+      0.9345619658119658,
+      1.1618589743589745,
+      0.7745726495726496,
+      0.4511217948717949
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.0054666257031572175,
+      0.01767145235041822,
+      0.04688230491681166,
+      0.09368004613712308,
+      0.14996995613910258
+    ],
+    "infer_info_gains": [
+      1.2973813737865565,
+      1.1859035042362127,
+      0.9568037366593334,
+      0.6996948701516982,
+      0.5142123508702605
+    ],
+    "infer_delta_probs": [
+      0.003866573204128634,
+      0.012204826647261004,
+      0.02921085256639344,
+      0.04679774122031141,
+      0.056289910001979505
+    ],
+    "infer_expected_reward_accs": [
+      0.3157051282051282,
+      0.2532051282051282,
+      0.4551282051282051,
+      0.6891025641025641,
+      0.8397435897435898
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      0.897250544460723,
+      0.6287653042377455,
+      0.45063194489436326,
+      0.3534822055934681,
+      0.2941118773114847
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5833333333333336,
+      0.7100000000000001,
+      0.7920833333333339,
+      0.834871794871795,
+      0.8624038461538464
+    ],
+    "infer_expected_random_set_idxs": [
+      0.9214743589743589,
+      1.080128205128205,
+      1.467147435897436,
+      1.0793269230769231,
+      0.9951923076923077,
+      1.2128739316239316,
+      1.2235576923076923,
+      0.8221153846153846,
+      1.267094017094017,
+      1.2689636752136753,
+      0.7694978632478633,
+      1.1829594017094016,
+      1.200587606837607,
+      0.6923076923076923,
+      0.9620726495726496,
+      1.1792200854700854,
+      1.0491452991452992,
+      1.014690170940171,
+      0.5491452991452992,
+      0.6655982905982906,
+      1.1017628205128205,
+      0.9858440170940171,
+      0.5491452991452992,
+      0.7430555555555556,
+      0.9428418803418803,
+      1.0341880341880343,
+      0.8474893162393162,
+      1.1826923076923077,
+      0.8816773504273504,
+      0.7865918803418803,
+      1.1599893162393162,
+      0.7601495726495726,
+      1.0980235042735043,
+      0.7844551282051282,
+      0.9743589743589743,
+      0.7254273504273504,
+      0.6712072649572649,
+      0.655715811965812,
+      0.9706196581196581,
+      1.0368589743589745,
+      1.0990918803418803,
+      0.7630876068376068,
+      0.6813568376068376,
+      0.7962072649572649,
+      0.7077991452991453,
+      0.6821581196581197,
+      0.8894230769230769,
+      1.0870726495726495,
+      1.4126602564102564,
+      1.080662393162393,
+      0.6736111111111112,
+      0.9503205128205128,
+      1.3627136752136753,
+      1.2270299145299146,
+      0.7029914529914529,
+      1.202457264957265,
+      0.5876068376068376,
+      0.8571047008547008,
+      0.8373397435897436,
+      1.0352564102564104,
+      0.8485576923076923,
+      1.1156517094017093,
+      1.1017628205128205,
+      0.9428418803418803,
+      1.1490384615384615,
+      0.8311965811965812,
+      0.9660790598290598,
+      0.5878739316239316,
+      1.2625534188034189,
+      1.2379807692307692,
+      0.750801282051282,
+      1.0665064102564104,
+      1.359241452991453,
+      0.9834401709401709,
+      1.2999465811965811,
+      1.1172542735042734,
+      1.0259081196581197,
+      1.2278311965811965,
+      0.8218482905982906,
+      0.9874465811965812,
+      1.2142094017094016,
+      0.6957799145299145,
+      1.3010149572649572,
+      1.0667735042735043,
+      1.1044337606837606,
+      1.138621794871795,
+      0.6680021367521367,
+      0.9385683760683761,
+      1.2673611111111112,
+      0.8269230769230769,
+      0.6324786324786325,
+      1.219284188034188,
+      1.467681623931624,
+      0.8352029914529915,
+      0.8458867521367521,
+      1.076655982905983,
+      1.0024038461538463,
+      1.1111111111111112,
+      0.7596153846153846,
+      0.5600961538461539
+    ],
+    "kls": [
+      0.0,
+      1.5822897398973041,
+      2.753687006972073,
+      3.5943138557137027,
+      4.166582973588018,
+      4.476869997653859
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.4437660256410258,
+      0.4619230769230768,
+      0.4820673076923077,
+      0.49786858974358966,
+      0.513814102564103
+    ],
+    "model_random_set_direct_idxs": [
+      1.1327457264957266,
+      1.155715811965812,
+      1.421474358974359,
+      1.3322649572649572,
+      1.0552884615384615,
+      0.6303418803418803,
+      0.7548076923076923,
+      0.7652243589743589,
+      1.0384615384615385,
+      1.5128205128205128,
+      1.2932692307692308,
+      0.8306623931623932,
+      1.3696581196581197,
+      0.8880876068376068,
+      1.404647435897436,
+      1.453258547008547,
+      1.2019230769230769,
+      0.9917200854700855,
+      0.7548076923076923,
+      0.9155982905982906,
+      0.6498397435897436,
+      1.1674679487179487,
+      0.8514957264957265,
+      1.2139423076923077,
+      0.9150641025641025,
+      0.8896901709401709,
+      1.063034188034188,
+      1.3242521367521367,
+      0.7411858974358975,
+      1.2721688034188035,
+      1.373397435897436,
+      1.2831196581196582,
+      0.46047008547008544,
+      1.3747329059829059,
+      0.8587072649572649,
+      1.1052350427350428,
+      0.7927350427350427,
+      0.8223824786324786,
+      1.0579594017094016,
+      0.7283653846153846,
+      1.2240918803418803,
+      1.311965811965812,
+      0.7577457264957265,
+      1.1199252136752136,
+      1.1156517094017093,
+      0.6169871794871795,
+      0.8258547008547008,
+      1.361912393162393,
+      1.2729700854700854,
+      1.0277777777777777,
+      1.2099358974358974,
+      0.8189102564102564,
+      1.1810897435897436,
+      1.2932692307692308,
+      1.092681623931624,
+      1.2686965811965811,
+      0.8696581196581197,
+      0.719551282051282,
+      1.2793803418803418,
+      0.9770299145299145,
+      1.076655982905983,
+      0.7732371794871795,
+      1.1770833333333333,
+      0.6794871794871795,
+      1.3098290598290598,
+      0.9155982905982906,
+      0.9861111111111112,
+      0.42761752136752135,
+      0.8170405982905983,
+      1.1161858974358974,
+      1.0304487179487178,
+      0.7131410256410257,
+      1.234775641025641,
+      0.7767094017094017,
+      1.2104700854700854,
+      0.7470619658119658,
+      0.7459935897435898,
+      1.0550213675213675,
+      0.8141025641025641,
+      1.2064636752136753,
+      1.1463675213675213,
+      0.8234508547008547,
+      1.0520833333333333,
+      0.9778311965811965,
+      1.1754807692307692,
+      1.0889423076923077,
+      0.6789529914529915,
+      0.7708333333333334,
+      1.046474358974359,
+      1.233974358974359,
+      0.47622863247863245,
+      1.0590277777777777,
+      1.3298611111111112,
+      1.1746794871794872,
+      1.3103632478632479,
+      0.6728098290598291,
+      0.9767628205128205,
+      0.4270833333333333,
+      0.5082799145299145,
+      0.7131410256410257
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7152403846153843,
+      0.7165064102564103,
+      0.7290064102564108,
+      0.734182692307692,
+      0.7274038461538463
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.6607852564102561,
+      0.666682692307692,
+      0.6780608974358974,
+      0.6833814102564103,
+      0.6776762820512823
+    ]
+  }
+}

results_v2/flight_uniform_kmin_89_seed1/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_89_seed1.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.014423076923076924,
+      0.25801282051282054,
+      0.3557692307692308,
+      0.40224358974358976,
+      0.39903846153846156
+    ],
+    "model_accs": [
+      0.014423076923076924,
+      0.25801282051282054,
+      0.3557692307692308,
+      0.40224358974358976,
+      0.39903846153846156
+    ],
+    "model_reward_accs": [
+      0.22916666666666666,
+      0.2948717948717949,
+      0.3605769230769231,
+      0.3701923076923077,
+      0.40064102564102566
+    ],
+    "model_consistencies": [
+      0.41346153846153844,
+      0.6362179487179487,
+      0.6602564102564102,
+      0.6810897435897436,
+      0.6778846153846154
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.1173317302720653,
+      1.0955683364394597,
+      1.0577657980432393,
+      1.0443507493455855,
+      0.9979551764286294
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.46955128205128216,
+      0.4787820512820517,
+      0.5028685897435897,
+      0.5072435897435901,
+      0.5262339743589742
+    ],
+    "model_random_set_idxs": [
+      0.7302350427350427,
+      1.2475961538461537,
+      1.2160790598290598,
+      1.1001602564102564,
+      1.0048076923076923,
+      1.1036324786324787,
+      1.1842948717948718,
+      0.9839743589743589,
+      1.0753205128205128,
+      1.111378205128205,
+      0.8485576923076923,
+      1.0868055555555556,
+      1.2110042735042734,
+      0.8279914529914529,
+      1.049412393162393,
+      1.1594551282051282,
+      0.9198717948717948,
+      0.84375,
+      0.7473290598290598,
+      0.7043269230769231,
+      1.125534188034188,
+      1.015491452991453,
+      0.7168803418803419,
+      0.9393696581196581,
+      1.202457264957265,
+      0.9946581196581197,
+      0.9802350427350427,
+      1.1901709401709402,
+      0.7126068376068376,
+      1.015224358974359,
+      1.0566239316239316,
+      1.0638354700854702,
+      0.9396367521367521,
+      1.001068376068376,
+      0.9182692307692307,
+      0.8587072649572649,
+      0.5013354700854701,
+      0.9147970085470085,
+      1.1661324786324787,
+      0.968482905982906,
+      1.124465811965812,
+      0.9791666666666666,
+      0.7422542735042735,
+      0.9169337606837606,
+      0.9180021367521367,
+      0.6375534188034188,
+      0.7387820512820513,
+      1.3808760683760684,
+      1.4014423076923077,
+      1.1076388888888888,
+      0.9286858974358975,
+      1.1992521367521367,
+      1.1802884615384615,
+      1.2473290598290598,
+      0.6720085470085471,
+      1.342681623931624,
+      0.6452991452991453,
+      0.7513354700854701,
+      0.937232905982906,
+      1.0598290598290598,
+      1.0486111111111112,
+      0.7112713675213675,
+      1.0128205128205128,
+      1.0408653846153846,
+      1.3717948717948718,
+      1.0854700854700854,
+      1.09375,
+      0.6428952991452992,
+      1.0683760683760684,
+      1.0862713675213675,
+      0.7414529914529915,
+      1.0341880341880343,
+      1.125,
+      0.8410790598290598,
+      1.1762820512820513,
+      1.0809294871794872,
+      1.0627670940170941,
+      1.2142094017094016,
+      1.123397435897436,
+      1.2577457264957266,
+      0.969551282051282,
+      0.8167735042735043,
+      1.2481303418803418,
+      1.2270299145299146,
+      0.9230769230769231,
+      1.15625,
+      0.7248931623931624,
+      0.6832264957264957,
+      1.1028311965811965,
+      1.046741452991453,
+      0.9257478632478633,
+      1.1346153846153846,
+      1.3565705128205128,
+      1.0408653846153846,
+      0.9759615384615384,
+      1.1164529914529915,
+      0.8611111111111112,
+      0.8840811965811965,
+      0.8084935897435898,
+      0.6119123931623932
+    ],
+    "model_probs": [
+      0.0016000524990285835,
+      0.002645477259897269,
+      0.0028933283154661017,
+      0.0029800806403862943,
+      0.003431083499748684,
+      0.0038786107379528345
+    ],
+    "model_info_gains": [
+      0.560108085918886,
+      -0.18869598655077646,
+      -0.10809472322304928,
+      -0.04587007420080057,
+      0.18458677459941233
+    ],
+    "model_delta_probs": [
+      0.0010454247608686856,
+      0.0002478510555688328,
+      8.675232492019237e-05,
+      0.00045100285936238983,
+      0.0004475272382041504
+    ],
+    "model_expected_reward_accs": [
+      0.22916666666666666,
+      0.27403846153846156,
+      0.3701923076923077,
+      0.3974358974358974,
+      0.40865384615384615
+    ],
+    "model_expected_consistencies": [
+      0.4230769230769231,
+      0.7131410256410257,
+      0.7419871794871795,
+      0.7371794871794872,
+      0.7259615384615384
+    ],
+    "model_expected_l2s": [
+      1.356185474447443,
+      1.073098156332158,
+      1.0488450665985527,
+      1.023869246984784,
+      0.9999997536947172,
+      0.9438793919608606
+    ],
+    "model_expected_random_set_accs": [
+      0.34657051282051293,
+      0.488429487179487,
+      0.5061538461538461,
+      0.5195032051282054,
+      0.5273878205128215,
+      0.5537339743589741
+    ],
+    "model_expected_random_set_idxs": [
+      0.9834401709401709,
+      1.2956730769230769,
+      1.171741452991453,
+      1.0397970085470085,
+      1.0216346153846154,
+      1.092681623931624,
+      1.2705662393162394,
+      0.9310897435897436,
+      1.1968482905982907,
+      1.2804487179487178,
+      0.9730235042735043,
+      1.09375,
+      1.1663995726495726,
+      0.8691239316239316,
+      1.0424679487179487,
+      1.219284188034188,
+      0.9770299145299145,
+      0.8368055555555556,
+      0.7291666666666666,
+      0.7636217948717948,
+      1.1861645299145298,
+      1.0595619658119657,
+      0.7358440170940171,
+      0.9361645299145299,
+      1.186431623931624,
+      0.9604700854700855,
+      0.9706196581196581,
+      1.1957799145299146,
+      0.7037927350427351,
+      1.0379273504273505,
+      1.0376602564102564,
+      1.0114850427350428,
+      0.9869123931623932,
+      1.0838675213675213,
+      0.8720619658119658,
+      0.9449786324786325,
+      0.6615918803418803,
+      0.8691239316239316,
+      1.1826923076923077,
+      0.9927884615384616,
+      1.1012286324786325,
+      0.9727564102564102,
+      0.8215811965811965,
+      1.0018696581196582,
+      0.9505876068376068,
+      0.6941773504273504,
+      0.7516025641025641,
+      1.3368055555555556,
+      1.3314636752136753,
+      1.032318376068376,
+      0.9468482905982906,
+      1.1765491452991452,
+      1.1466346153846154,
+      1.173878205128205,
+      0.6621260683760684,
+      1.3154380341880343,
+      0.7387820512820513,
+      0.7521367521367521,
+      0.9163995726495726,
+      1.0600961538461537,
+      1.0098824786324787,
+      1.1081730769230769,
+      1.0841346153846154,
+      1.094284188034188,
+      1.3028846153846154,
+      0.9276175213675214,
+      1.078792735042735,
+      0.453525641025641,
+      1.064102564102564,
+      1.1351495726495726,
+      0.7203525641025641,
+      1.0993589743589745,
+      1.078525641025641,
+      0.8651175213675214,
+      1.140758547008547,
+      1.0344551282051282,
+      1.0379273504273505,
+      1.2751068376068375,
+      0.9375,
+      1.203792735042735,
+      1.0456730769230769,
+      0.813034188034188,
+      1.2764423076923077,
+      1.1904380341880343,
+      0.8688568376068376,
+      1.0763888888888888,
+      0.7313034188034188,
+      0.7612179487179487,
+      1.1071047008547008,
+      0.9620726495726496,
+      0.6861645299145299,
+      1.1623931623931625,
+      1.3397435897435896,
+      1.0088141025641026,
+      0.9644764957264957,
+      1.0058760683760684,
+      0.9455128205128205,
+      0.9289529914529915,
+      0.8942307692307693,
+      0.5969551282051282
+    ],
+    "infer_reward_accs": [
+      0.22756410256410256,
+      0.28846153846153844,
+      0.33653846153846156,
+      0.48717948717948717,
+      0.6153846153846154
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.0364009009613564,
+      0.6758453087587981,
+      0.39267410751462617,
+      0.23565255857733042,
+      0.12117800978710143
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.5169391025641026,
+      0.6887500000000006,
+      0.8116826923076927,
+      0.87996794871795,
+      0.9300320512820546
+    ],
+    "infer_random_set_idxs": [
+      0.7305021367521367,
+      1.046474358974359,
+      1.498397435897436,
+      1.0988247863247864,
+      0.9975961538461539,
+      1.2454594017094016,
+      1.1912393162393162,
+      0.8581730769230769,
+      1.1688034188034189,
+      1.2227564102564104,
+      0.6899038461538461,
+      1.2524038461538463,
+      1.2264957264957266,
+      0.625267094017094,
+      0.9246794871794872,
+      1.1829594017094016,
+      1.1180555555555556,
+      0.9826388888888888,
+      0.5259081196581197,
+      0.6359508547008547,
+      1.1340811965811965,
+      1.015224358974359,
+      0.5296474358974359,
+      0.7350427350427351,
+      0.9294871794871795,
+      1.1346153846153846,
+      0.8058226495726496,
+      1.1786858974358974,
+      0.8830128205128205,
+      0.750267094017094,
+      1.1370192307692308,
+      0.7307692307692307,
+      1.1781517094017093,
+      0.7569444444444444,
+      0.9636752136752137,
+      0.6223290598290598,
+      0.5494123931623932,
+      0.6212606837606838,
+      0.9407051282051282,
+      1.045940170940171,
+      1.1909722222222223,
+      0.7935363247863247,
+      0.6591880341880342,
+      0.7021901709401709,
+      0.6623931623931624,
+      0.625,
+      0.8512286324786325,
+      1.1076388888888888,
+      1.4644764957264957,
+      1.0897435897435896,
+      0.6132478632478633,
+      0.9142628205128205,
+      1.421207264957265,
+      1.203258547008547,
+      0.6797542735042735,
+      1.2467948717948718,
+      0.5411324786324786,
+      0.8362713675213675,
+      0.906784188034188,
+      1.0248397435897436,
+      0.8344017094017094,
+      0.7323717948717948,
+      1.0980235042735043,
+      0.9158653846153846,
+      1.15625,
+      0.905448717948718,
+      0.9286858974358975,
+      0.7446581196581197,
+      1.2104700854700854,
+      1.2529380341880343,
+      0.7080662393162394,
+      1.0910790598290598,
+      1.4134615384615385,
+      0.9553952991452992,
+      1.3381410256410255,
+      1.1314102564102564,
+      1.0715811965811965,
+      1.2288995726495726,
+      0.9861111111111112,
+      1.0101495726495726,
+      1.2251602564102564,
+      0.7102029914529915,
+      1.3020833333333333,
+      1.0173611111111112,
+      1.0683760683760684,
+      1.2398504273504274,
+      0.6712072649572649,
+      0.9022435897435898,
+      1.297542735042735,
+      0.9869123931623932,
+      0.8234508547008547,
+      1.1883012820512822,
+      1.5133547008547008,
+      0.8680555555555556,
+      0.8565705128205128,
+      1.1143162393162394,
+      0.9797008547008547,
+      1.1049679487179487,
+      0.7662927350427351,
+      0.4906517094017094
+    ],
+    "infer_probs": [
+      0.0016000524990285835,
+      0.006452993370121709,
+      0.031016101405699513,
+      0.11879874641985785,
+      0.286715052849337,
+      0.5169887000038915
+    ],
+    "infer_info_gains": [
+      1.462310675254804,
+      1.5563830497094668,
+      1.3053320795567755,
+      0.9736780494050666,
+      0.7438049032738578
+    ],
+    "infer_delta_probs": [
+      0.004852940871093126,
+      0.024563108035577805,
+      0.08778264501415833,
+      0.16791630642947916,
+      0.23027364715455453
+    ],
+    "infer_expected_reward_accs": [
+      0.22916666666666666,
+      0.12980769230769232,
+      0.22115384615384615,
+      0.4423076923076923,
+      0.6682692307692307
+    ],
+    "infer_expected_l2s": [
+      1.356185474447443,
+      0.872787909160532,
+      0.5295420492438481,
+      0.3269313333910711,
+      0.2190666745756778,
+      0.12107033323392273
+    ],
+    "infer_expected_random_set_accs": [
+      0.34657051282051293,
+      0.5928044871794876,
+      0.754903846153847,
+      0.8462339743589748,
+      0.8937019230769246,
+      0.9358173076923096
+    ],
+    "infer_expected_random_set_idxs": [
+      0.9481837606837606,
+      1.0830662393162394,
+      1.4473824786324787,
+      1.0737179487179487,
+      0.9930555555555556,
+      1.1909722222222223,
+      1.1968482905982907,
+      0.8418803418803419,
+      1.251068376068376,
+      1.2529380341880343,
+      0.8135683760683761,
+      1.156784188034188,
+      1.1933760683760684,
+      0.7280982905982906,
+      0.9783653846153846,
+      1.1642628205128205,
+      1.0333867521367521,
+      1.0109508547008548,
+      0.5510149572649573,
+      0.6709401709401709,
+      1.1038995726495726,
+      0.9647435897435898,
+      0.5667735042735043,
+      0.7788461538461539,
+      0.9500534188034188,
+      1.031784188034188,
+      0.8311965811965812,
+      1.1522435897435896,
+      0.8827457264957265,
+      0.8098290598290598,
+      1.1826923076923077,
+      0.7884615384615384,
+      1.076655982905983,
+      0.8175747863247863,
+      0.9455128205128205,
+      0.7483974358974359,
+      0.7064636752136753,
+      0.6688034188034188,
+      1.0045405982905984,
+      1.0291132478632479,
+      1.0705128205128205,
+      0.8114316239316239,
+      0.6840277777777778,
+      0.8271901709401709,
+      0.7395833333333334,
+      0.7051282051282052,
+      0.8819444444444444,
+      1.0849358974358974,
+      1.4153311965811965,
+      1.0499465811965811,
+      0.6992521367521367,
+      0.9532585470085471,
+      1.3402777777777777,
+      1.2403846153846154,
+      0.7230235042735043,
+      1.1829594017094016,
+      0.6148504273504274,
+      0.8368055555555556,
+      0.8827457264957265,
+      1.044871794871795,
+      0.8624465811965812,
+      1.1295405982905984,
+      1.1028311965811965,
+      0.9308226495726496,
+      1.1808226495726495,
+      0.8341346153846154,
+      0.9708867521367521,
+      0.5908119658119658,
+      1.2315705128205128,
+      1.2288995726495726,
+      0.7753739316239316,
+      1.0405982905982907,
+      1.328792735042735,
+      1.0048076923076923,
+      1.2775106837606838,
+      1.09375,
+      1.0438034188034189,
+      1.204059829059829,
+      0.8165064102564102,
+      0.9909188034188035,
+      1.2088675213675213,
+      0.6968482905982906,
+      1.2633547008547008,
+      1.0868055555555556,
+      1.0833333333333333,
+      1.154647435897436,
+      0.6904380341880342,
+      0.9441773504273504,
+      1.2556089743589745,
+      0.8496260683760684,
+      0.625,
+      1.1915064102564104,
+      1.4334935897435896,
+      0.8787393162393162,
+      0.8739316239316239,
+      1.0520833333333333,
+      1.0133547008547008,
+      1.0673076923076923,
+      0.7644230769230769,
+      0.5667735042735043
+    ],
+    "kls": [
+      0.0,
+      1.739410347007777,
+      3.3451427358695605,
+      4.55259344199647,
+      5.405940411331798,
+      5.870099389156925
+    ],
+    "model_random_set_direct_accs": [
+      0.39953525641025617,
+      0.44679487179487126,
+      0.44769230769230745,
+      0.46693910256410237,
+      0.477724358974359,
+      0.49424679487179457
+    ],
+    "model_random_set_direct_idxs": [
+      1.1549145299145298,
+      1.1778846153846154,
+      1.405715811965812,
+      1.3392094017094016,
+      1.0339209401709402,
+      0.6338141025641025,
+      0.7839209401709402,
+      0.7534722222222222,
+      1.0280448717948718,
+      1.4941239316239316,
+      1.3269230769230769,
+      0.8624465811965812,
+      1.3699252136752136,
+      0.9174679487179487,
+      1.4145299145299146,
+      1.4719551282051282,
+      1.200587606837607,
+      1.0037393162393162,
+      0.7454594017094017,
+      0.9166666666666666,
+      0.6498397435897436,
+      1.1533119658119657,
+      0.8416132478632479,
+      1.2580128205128205,
+      0.9599358974358975,
+      0.8950320512820513,
+      1.0600961538461537,
+      1.3416132478632479,
+      0.7094017094017094,
+      1.280715811965812,
+      1.407852564102564,
+      1.2836538461538463,
+      0.44925213675213677,
+      1.3482905982905984,
+      0.8547008547008547,
+      1.1004273504273505,
+      0.8579059829059829,
+      0.8082264957264957,
+      1.048878205128205,
+      0.6794871794871795,
+      1.2219551282051282,
+      1.2852564102564104,
+      0.7702991452991453,
+      1.1193910256410255,
+      1.0940170940170941,
+      0.6380876068376068,
+      0.8199786324786325,
+      1.3667200854700854,
+      1.265491452991453,
+      1.0053418803418803,
+      1.202991452991453,
+      0.8458867521367521,
+      1.2003205128205128,
+      1.3044871794871795,
+      1.1081730769230769,
+      1.2473290598290598,
+      0.8536324786324786,
+      0.7414529914529915,
+      1.2927350427350428,
+      0.9519230769230769,
+      1.0844017094017093,
+      0.7657585470085471,
+      1.1594551282051282,
+      0.6418269230769231,
+      1.3181089743589745,
+      0.8966346153846154,
+      0.9823717948717948,
+      0.4270833333333333,
+      0.7574786324786325,
+      1.0966880341880343,
+      1.0411324786324787,
+      0.6984508547008547,
+      1.2606837606837606,
+      0.8167735042735043,
+      1.1677350427350428,
+      0.75,
+      0.7203525641025641,
+      1.0392628205128205,
+      0.8538995726495726,
+      1.1850961538461537,
+      1.1145833333333333,
+      0.8100961538461539,
+      1.047008547008547,
+      0.9826388888888888,
+      1.1821581196581197,
+      1.1378205128205128,
+      0.6826923076923077,
+      0.7620192307692307,
+      1.061431623931624,
+      1.2409188034188035,
+      0.4722222222222222,
+      1.0077457264957266,
+      1.3253205128205128,
+      1.1877670940170941,
+      1.3226495726495726,
+      0.6650641025641025,
+      1.0066773504273505,
+      0.44177350427350426,
+      0.4890491452991453,
+      0.6981837606837606
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3600000000000056,
+      0.7173878205128209,
+      0.7198076923076925,
+      0.7192467948717951,
+      0.7231249999999998,
+      0.7228044871794868
+    ],
+    "model_random_set_consistencies": [
+      0.37543269230769244,
+      0.6572916666666669,
+      0.6706250000000002,
+      0.6686698717948719,
+      0.673044871794872,
+      0.6773076923076922
+    ]
+  }
+}

results_v2/flight_uniform_kmin_96_seed0/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/u/linluqiu/bayesian_inference/eval_data/prior_uniform_kmin_96_seed0.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.003205128205128205,
+      0.2516025641025641,
+      0.3541666666666667,
+      0.33974358974358976,
+      0.421474358974359
+    ],
+    "model_accs": [
+      0.003205128205128205,
+      0.2516025641025641,
+      0.3541666666666667,
+      0.33974358974358976,
+      0.421474358974359
+    ],
+    "model_reward_accs": [
+      0.23237179487179488,
+      0.22756410256410256,
+      0.34935897435897434,
+      0.33653846153846156,
+      0.44871794871794873
+    ],
+    "model_consistencies": [
+      0.3717948717948718,
+      0.6298076923076923,
+      0.6698717948717948,
+      0.7163461538461539,
+      0.6538461538461539
+    ],
+    "model_l2s": [
+      1.355730195859342,
+      1.1037068398337326,
+      1.0764533000983703,
+      1.080787462277122,
+      1.0337530856564145,
+      0.9749490194271498
+    ],
+    "model_random_set_accs": [
+      0.3485096153846155,
+      0.47735576923076933,
+      0.48870192307692345,
+      0.48964743589743537,
+      0.5113942307692307,
+      0.5378205128205128
+    ],
+    "model_random_set_idxs": [
+      0.7534722222222222,
+      1.2574786324786325,
+      1.2061965811965811,
+      1.110309829059829,
+      1.0211004273504274,
+      1.0916132478632479,
+      1.1848290598290598,
+      0.969551282051282,
+      1.063568376068376,
+      1.0857371794871795,
+      0.8661858974358975,
+      1.0862713675213675,
+      1.2262286324786325,
+      0.8373397435897436,
+      1.0480769230769231,
+      1.173878205128205,
+      0.9300213675213675,
+      0.844017094017094,
+      0.7326388888888888,
+      0.6987179487179487,
+      1.1383547008547008,
+      1.0251068376068375,
+      0.7136752136752137,
+      0.9254807692307693,
+      1.1971153846153846,
+      0.9922542735042735,
+      0.9751602564102564,
+      1.169871794871795,
+      0.7000534188034188,
+      1.0080128205128205,
+      1.0809294871794872,
+      1.076655982905983,
+      0.9318910256410257,
+      0.9978632478632479,
+      0.9086538461538461,
+      0.8736645299145299,
+      0.5235042735042735,
+      0.9337606837606838,
+      1.172275641025641,
+      0.9580662393162394,
+      1.109508547008547,
+      0.969284188034188,
+      0.7483974358974359,
+      0.9337606837606838,
+      0.9206730769230769,
+      0.6621260683760684,
+      0.7521367521367521,
+      1.3629807692307692,
+      1.3814102564102564,
+      1.0814636752136753,
+      0.9375,
+      1.2069978632478633,
+      1.1789529914529915,
+      1.268162393162393,
+      0.6821581196581197,
+      1.3218482905982907,
+      0.6485042735042735,
+      0.7801816239316239,
+      0.9708867521367521,
+      1.0723824786324787,
+      1.0502136752136753,
+      0.7131410256410257,
+      0.9978632478632479,
+      1.0357905982905984,
+      1.3627136752136753,
+      1.0643696581196582,
+      1.108707264957265,
+      0.6626602564102564,
+      1.0838675213675213,
+      1.0643696581196582,
+      0.7654914529914529,
+      1.0253739316239316,
+      1.1199252136752136,
+      0.8597756410256411,
+      1.1842948717948718,
+      1.0819978632478633,
+      1.0571581196581197,
+      1.2112713675213675,
+      1.109775641025641,
+      1.233707264957265,
+      0.9831730769230769,
+      0.8108974358974359,
+      1.2417200854700854,
+      1.2110042735042734,
+      0.9425747863247863,
+      1.1749465811965811,
+      0.7403846153846154,
+      0.7035256410256411,
+      1.0964209401709402,
+      1.079594017094017,
+      0.9137286324786325,
+      1.1132478632478633,
+      1.3394764957264957,
+      1.0582264957264957,
+      0.9674145299145299,
+      1.1012286324786325,
+      0.8461538461538461,
+      0.875534188034188,
+      0.8143696581196581,
+      0.624732905982906
+    ],
+    "model_probs": [
+      0.0016000823995457908,
+      0.002714090449183587,
+      0.0028797199345283876,
+      0.0025663436431875982,
+      0.003157137573290323,
+      0.004076753222832052
+    ],
+    "model_info_gains": [
+      0.5056572921464129,
+      -0.14230384342242844,
+      -0.22469083594232972,
+      0.11495815219395365,
+      0.24443845611212126
+    ],
+    "model_delta_probs": [
+      0.0011140080496377964,
+      0.00016562948534480044,
+      -0.00031337629134078963,
+      0.0005907939301027254,
+      0.0009196156495417285
+    ],
+    "model_expected_reward_accs": [
+      0.23397435897435898,
+      0.22756410256410256,
+      0.33974358974358976,
+      0.3108974358974359,
+      0.44551282051282054
+    ],
+    "model_expected_consistencies": [
+      0.3798076923076923,
+      0.6987179487179487,
+      0.7259615384615384,
+      0.7676282051282052,
+      0.7419871794871795
+    ],
+    "model_expected_l2s": [
+      1.3561714894324812,
+      1.0726618911260484,
+      1.0524155611566373,
+      1.0541008130441345,
+      0.9977826983851278,
+      0.931137968329759
+    ],
+    "model_expected_random_set_accs": [
+      0.34895833333333326,
+      0.49089743589743606,
+      0.5000160256410258,
+      0.5052724358974354,
+      0.5306249999999998,
+      0.5596634615384616
+    ],
+    "model_expected_random_set_idxs": [
+      0.8362713675213675,
+      1.3026175213675213,
+      1.171741452991453,
+      1.0689102564102564,
+      1.0333867521367521,
+      1.0884081196581197,
+      1.2502670940170941,
+      0.9305555555555556,
+      1.1883012820512822,
+      1.2403846153846154,
+      0.9901175213675214,
+      1.1014957264957266,
+      1.1760149572649572,
+      0.8677884615384616,
+      1.0539529914529915,
+      1.2168803418803418,
+      0.9847756410256411,
+      0.8325320512820513,
+      0.7110042735042735,
+      0.7406517094017094,
+      1.1754807692307692,
+      1.0670405982905984,
+      0.7272970085470085,
+      0.9241452991452992,
+      1.1805555555555556,
+      0.9428418803418803,
+      0.9815705128205128,
+      1.1792200854700854,
+      0.6816239316239316,
+      1.0352564102564104,
+      1.079059829059829,
+      1.0077457264957266,
+      0.9762286324786325,
+      1.0822649572649572,
+      0.8723290598290598,
+      0.9586004273504274,
+      0.6581196581196581,
+      0.8771367521367521,
+      1.1909722222222223,
+      0.9724893162393162,
+      1.0908119658119657,
+      0.9751602564102564,
+      0.8245192307692307,
+      0.9983974358974359,
+      0.9703525641025641,
+      0.7158119658119658,
+      0.7561431623931624,
+      1.3298611111111112,
+      1.329059829059829,
+      0.9962606837606838,
+      0.9564636752136753,
+      1.1760149572649572,
+      1.126068376068376,
+      1.1960470085470085,
+      0.6637286324786325,
+      1.3018162393162394,
+      0.7291666666666666,
+      0.7996794871794872,
+      0.9423076923076923,
+      1.0857371794871795,
+      0.9834401709401709,
+      1.1033653846153846,
+      1.0753205128205128,
+      1.0849358974358974,
+      1.3215811965811965,
+      0.9217414529914529,
+      1.0916132478632479,
+      0.4895833333333333,
+      1.0649038461538463,
+      1.1060363247863247,
+      0.7459935897435898,
+      1.0870726495726495,
+      1.0715811965811965,
+      0.8800747863247863,
+      1.1503739316239316,
+      1.0440705128205128,
+      1.0411324786324787,
+      1.2743055555555556,
+      0.9182692307692307,
+      1.1658653846153846,
+      1.0432692307692308,
+      0.8215811965811965,
+      1.2887286324786325,
+      1.173344017094017,
+      0.9049145299145299,
+      1.108974358974359,
+      0.7516025641025641,
+      0.7873931623931624,
+      1.0929487179487178,
+      0.9946581196581197,
+      0.6738782051282052,
+      1.1506410256410255,
+      1.330662393162393,
+      1.0269764957264957,
+      0.9375,
+      0.9738247863247863,
+      0.9185363247863247,
+      0.9313568376068376,
+      0.9001068376068376,
+      0.6084401709401709
+    ],
+    "infer_reward_accs": [
+      0.21794871794871795,
+      0.18269230769230768,
+      0.28685897435897434,
+      0.46153846153846156,
+      0.8076923076923077
+    ],
+    "infer_l2s": [
+      1.355730195859342,
+      1.0171382911008275,
+      0.55641163215407,
+      0.2931302433476068,
+      0.08745336062694564,
+      0.0068469139621978444
+    ],
+    "infer_random_set_accs": [
+      0.3495993589743591,
+      0.5251602564102562,
+      0.7392147435897443,
+      0.8571955128205142,
+      0.9446794871794912,
+      0.9789583333333388
+    ],
+    "infer_random_set_idxs": [
+      0.7617521367521367,
+      1.046474358974359,
+      1.4794337606837606,
+      1.094818376068376,
+      1.0088141025641026,
+      1.2443910256410255,
+      1.1730769230769231,
+      0.8656517094017094,
+      1.1589209401709402,
+      1.2248931623931625,
+      0.7144764957264957,
+      1.2251602564102564,
+      1.2262286324786325,
+      0.6602564102564102,
+      0.9265491452991453,
+      1.1607905982905984,
+      1.0902777777777777,
+      0.9826388888888888,
+      0.5341880341880342,
+      0.6404914529914529,
+      1.1300747863247864,
+      1.015491452991453,
+      0.5544871794871795,
+      0.7411858974358975,
+      0.9340277777777778,
+      1.1129807692307692,
+      0.8002136752136753,
+      1.1594551282051282,
+      0.875801282051282,
+      0.7684294871794872,
+      1.1212606837606838,
+      0.750267094017094,
+      1.1316773504273505,
+      0.7649572649572649,
+      0.9556623931623932,
+      0.6658653846153846,
+      0.5854700854700855,
+      0.656517094017094,
+      0.9642094017094017,
+      1.0411324786324787,
+      1.1682692307692308,
+      0.8055555555555556,
+      0.6621260683760684,
+      0.7475961538461539,
+      0.6939102564102564,
+      0.6522435897435898,
+      0.8736645299145299,
+      1.0838675213675213,
+      1.4407051282051282,
+      1.0595619658119657,
+      0.6599893162393162,
+      0.9217414529914529,
+      1.3846153846153846,
+      1.2131410256410255,
+      0.7155448717948718,
+      1.2206196581196582,
+      0.5731837606837606,
+      0.843215811965812,
+      0.9284188034188035,
+      1.0528846153846154,
+      0.8629807692307693,
+      0.7617521367521367,
+      1.1001602564102564,
+      0.9174679487179487,
+      1.1768162393162394,
+      0.9206730769230769,
+      0.9471153846153846,
+      0.7625534188034188,
+      1.1997863247863247,
+      1.235042735042735,
+      0.7350427350427351,
+      1.0566239316239316,
+      1.3894230769230769,
+      0.9732905982905983,
+      1.3357371794871795,
+      1.1207264957264957,
+      1.0595619658119657,
+      1.2107371794871795,
+      0.9751602564102564,
+      0.9882478632478633,
+      1.231837606837607,
+      0.7011217948717948,
+      1.2841880341880343,
+      1.0397970085470085,
+      1.0841346153846154,
+      1.2003205128205128,
+      0.688034188034188,
+      0.9163995726495726,
+      1.2863247863247864,
+      1.015491452991453,
+      0.8135683760683761,
+      1.1805555555555556,
+      1.4871794871794872,
+      0.8661858974358975,
+      0.8632478632478633,
+      1.0918803418803418,
+      0.9855769230769231,
+      1.0846688034188035,
+      0.7582799145299145,
+      0.5122863247863247
+    ],
+    "infer_probs": [
+      0.0016000823995457908,
+      0.007790941677215484,
+      0.05550956894839123,
+      0.26503495151047946,
+      0.6410415736321832,
+      0.8462444793332953
+    ],
+    "infer_info_gains": [
+      1.6451403802015137,
+      1.9805816496768271,
+      1.5784404217936465,
+      1.1395799347398703,
+      0.42360379616882327
+    ],
+    "infer_delta_probs": [
+      0.006190859277669694,
+      0.04771862727117574,
+      0.20952538256208825,
+      0.37600662212170366,
+      0.20520290570111216
+    ],
+    "infer_expected_reward_accs": [
+      0.23397435897435898,
+      0.08333333333333333,
+      0.16506410256410256,
+      0.41346153846153844,
+      0.8044871794871795
+    ],
+    "infer_expected_l2s": [
+      1.3561714894324812,
+      0.8209371209890245,
+      0.4342893647041417,
+      0.2642344536699509,
+      0.09398022242825355,
+      0.011053884496756147
+    ],
+    "infer_expected_random_set_accs": [
+      0.34895833333333326,
+      0.6192307692307691,
+      0.794423076923077,
+      0.875993589743591,
+      0.9461057692307725,
+      0.9772916666666719
+    ],
+    "infer_expected_random_set_idxs": [
+      0.8007478632478633,
+      1.0849358974358974,
+      1.4329594017094016,
+      1.075587606837607,
+      1.0034722222222223,
+      1.185897435897436,
+      1.1810897435897436,
+      0.8405448717948718,
+      1.2390491452991452,
+      1.265491452991453,
+      0.8068910256410257,
+      1.1565170940170941,
+      1.1931089743589745,
+      0.7318376068376068,
+      0.9815705128205128,
+      1.1621260683760684,
+      1.03125,
+      0.9986645299145299,
+      0.5590277777777778,
+      0.6808226495726496,
+      1.1020299145299146,
+      0.9837072649572649,
+      0.5884081196581197,
+      0.7889957264957265,
+      0.9668803418803419,
+      1.0202991452991452,
+      0.8333333333333334,
+      1.1463675213675213,
+      0.8728632478632479,
+      0.8223824786324786,
+      1.1538461538461537,
+      0.8058226495726496,
+      1.076121794871795,
+      0.8287927350427351,
+      0.9401709401709402,
+      0.7585470085470085,
+      0.7061965811965812,
+      0.6931089743589743,
+      1.0168269230769231,
+      1.0320512820512822,
+      1.0731837606837606,
+      0.8218482905982906,
+      0.7019230769230769,
+      0.84375,
+      0.7390491452991453,
+      0.6952457264957265,
+      0.875267094017094,
+      1.0884081196581197,
+      1.3923611111111112,
+      1.0424679487179487,
+      0.7270299145299145,
+      0.9652777777777778,
+      1.328258547008547,
+      1.235042735042735,
+      0.7200854700854701,
+      1.1893696581196582,
+      0.6362179487179487,
+      0.8418803418803419,
+      0.8814102564102564,
+      1.0536858974358974,
+      0.8787393162393162,
+      1.1346153846153846,
+      1.108707264957265,
+      0.9345619658119658,
+      1.1813568376068375,
+      0.8496260683760684,
+      0.9644764957264957,
+      0.6004273504273504,
+      1.2256944444444444,
+      1.2313034188034189,
+      0.7828525641025641,
+      1.0357905982905984,
+      1.3242521367521367,
+      0.9869123931623932,
+      1.2620192307692308,
+      1.0860042735042734,
+      1.047542735042735,
+      1.2126068376068375,
+      0.8186431623931624,
+      0.9895833333333334,
+      1.1936431623931625,
+      0.6989850427350427,
+      1.2612179487179487,
+      1.076655982905983,
+      1.0982905982905984,
+      1.139957264957265,
+      0.6888354700854701,
+      0.9294871794871795,
+      1.2438568376068375,
+      0.8776709401709402,
+      0.6303418803418803,
+      1.1832264957264957,
+      1.4206730769230769,
+      0.8643162393162394,
+      0.8961004273504274,
+      1.0424679487179487,
+      1.0186965811965811,
+      1.0643696581196582,
+      0.7617521367521367,
+      0.5592948717948718
+    ],
+    "kls": [
+      0.0,
+      1.9471253552803973,
+      3.934564843088665,
+      5.418356160347691,
+      6.324039237384913,
+      6.405284566941408
+    ],
+    "model_random_set_direct_accs": [
+      0.3995833333333328,
+      0.4708493589743586,
+      0.45807692307692377,
+      0.4696314102564102,
+      0.4851442307692308,
+      0.5034294871794872
+    ],
+    "model_random_set_direct_idxs": [
+      1.1637286324786325,
+      1.172542735042735,
+      1.439102564102564,
+      1.3627136752136753,
+      1.0341880341880343,
+      0.6188568376068376,
+      0.8028846153846154,
+      0.7540064102564102,
+      1.0264423076923077,
+      1.4489850427350428,
+      1.3731303418803418,
+      0.8498931623931624,
+      1.4022435897435896,
+      0.9033119658119658,
+      1.4180021367521367,
+      1.5064102564102564,
+      1.1992521367521367,
+      1.0333867521367521,
+      0.7585470085470085,
+      0.9388354700854701,
+      0.6201923076923077,
+      1.1672008547008548,
+      0.8699252136752137,
+      1.2540064102564104,
+      0.9631410256410257,
+      0.8723290598290598,
+      0.8958333333333334,
+      1.3298611111111112,
+      0.6757478632478633,
+      1.3012820512820513,
+      1.4091880341880343,
+      1.2849893162393162,
+      0.4650106837606838,
+      1.3482905982905984,
+      0.8560363247863247,
+      1.063034188034188,
+      0.8490918803418803,
+      0.811965811965812,
+      1.2072649572649572,
+      0.6549145299145299,
+      1.171474358974359,
+      1.3234508547008548,
+      0.8146367521367521,
+      1.1463675213675213,
+      1.141292735042735,
+      0.6861645299145299,
+      0.8173076923076923,
+      1.3397435897435896,
+      1.2406517094017093,
+      0.9607371794871795,
+      1.232371794871795,
+      0.8202457264957265,
+      1.1450320512820513,
+      1.3357371794871795,
+      1.0905448717948718,
+      1.249465811965812,
+      0.9043803418803419,
+      0.7646901709401709,
+      1.3050213675213675,
+      0.9492521367521367,
+      1.077190170940171,
+      0.7633547008547008,
+      1.154647435897436,
+      0.624732905982906,
+      1.3298611111111112,
+      0.8784722222222222,
+      0.9863782051282052,
+      0.437232905982906,
+      0.7604166666666666,
+      0.7206196581196581,
+      1.0715811965811965,
+      0.6995192307692307,
+      1.3010149572649572,
+      0.6527777777777778,
+      1.1663995726495726,
+      0.7646901709401709,
+      0.7021901709401709,
+      1.0430021367521367,
+      0.8378739316239316,
+      1.1834935897435896,
+      1.1207264957264957,
+      0.8563034188034188,
+      1.0697115384615385,
+      0.9350961538461539,
+      1.2064636752136753,
+      1.1826923076923077,
+      0.7286324786324786,
+      0.7665598290598291,
+      1.0678418803418803,
+      1.2361111111111112,
+      0.45592948717948717,
+      0.9727564102564102,
+      1.3010149572649572,
+      1.2491987179487178,
+      1.326655982905983,
+      0.6431623931623932,
+      0.968215811965812,
+      0.44898504273504275,
+      0.5016025641025641,
+      0.7262286324786325
+    ],
+    "model_expected_random_set_consistencies": [
+      0.34000000000000286,
+      0.7172756410256409,
+      0.7224358974358974,
+      0.7179807692307693,
+      0.7163301282051283,
+      0.7309615384615381
+    ],
+    "model_random_set_consistencies": [
+      0.35543269230769375,
+      0.6499679487179483,
+      0.6679006410256406,
+      0.6649519230769231,
+      0.6647275641025637,
+      0.6772435897435896
+    ]
+  }
+}

results_v2/human_v2_5rounds/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/user/linluqiu/bayesian_inference/human_as_human_v2/human_v2_5rounds.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.36666666666666664,
+      0.35,
+      0.4166666666666667,
+      0.43,
+      0.37666666666666665
+    ],
+    "model_accs": [
+      0.36666666666666664,
+      0.35,
+      0.4166666666666667,
+      0.43,
+      0.37666666666666665
+    ],
+    "model_reward_accs": [
+      0.5733333333333334,
+      0.45,
+      0.39,
+      0.4266666666666667,
+      0.4266666666666667
+    ],
+    "model_consistencies": [
+      0.3333333333333333,
+      0.6433333333333333,
+      0.6766666666666666,
+      0.6933333333333334,
+      0.7
+    ],
+    "model_l2s": [
+      0.6798868004826828,
+      1.1674157163437613,
+      1.133358615660429,
+      1.1635329329151067,
+      1.1490561564388928,
+      1.0982486720063387
+    ],
+    "model_random_set_accs": [
+      0.6864666666666671,
+      0.4502,
+      0.47029999999999983,
+      0.45403333333333334,
+      0.4551999999999996,
+      0.48350000000000015
+    ],
+    "model_random_set_idxs": [
+      0.36833333333333335,
+      1.1861111111111111,
+      1.4833333333333334,
+      0.895,
+      0.8255555555555556,
+      1.258888888888889,
+      1.1455555555555557,
+      0.8455555555555555,
+      1.365,
+      1.3544444444444443,
+      0.5694444444444444,
+      1.1088888888888888,
+      1.0711111111111111,
+      0.7461111111111111,
+      0.965,
+      1.2538888888888888,
+      0.9672222222222222,
+      1.0988888888888888,
+      0.7155555555555555,
+      0.6838888888888889,
+      1.201111111111111,
+      0.7916666666666666,
+      0.6533333333333333,
+      0.8994444444444445,
+      1.038888888888889,
+      1.185,
+      0.9061111111111111,
+      1.403888888888889,
+      0.8811111111111111,
+      0.7283333333333334,
+      0.9872222222222222,
+      0.7461111111111111,
+      1.041111111111111,
+      0.7972222222222223,
+      1.1588888888888889,
+      0.5244444444444445,
+      0.26944444444444443,
+      0.5911111111111111,
+      1.2116666666666667,
+      1.1077777777777778,
+      1.2661111111111112,
+      0.6083333333333333,
+      0.5833333333333334,
+      0.675,
+      0.6016666666666667,
+      0.29777777777777775,
+      0.8205555555555556,
+      1.3222222222222222,
+      1.508888888888889,
+      1.241111111111111,
+      0.5233333333333333,
+      1.0677777777777777,
+      1.511111111111111,
+      1.0033333333333334,
+      0.4588888888888889,
+      1.3288888888888888,
+      0.37833333333333335,
+      0.7633333333333333,
+      0.7933333333333333,
+      1.02,
+      0.8177777777777778,
+      0.6922222222222222,
+      1.2305555555555556,
+      1.031111111111111,
+      1.1572222222222222,
+      1.093888888888889,
+      1.0333333333333334,
+      0.5983333333333334,
+      1.4544444444444444,
+      1.4922222222222221,
+      0.4172222222222222,
+      1.1111111111111112,
+      1.28,
+      0.9011111111111111,
+      1.298888888888889,
+      1.1188888888888888,
+      1.2566666666666666,
+      1.2361111111111112,
+      1.0377777777777777,
+      1.1272222222222221,
+      1.02,
+      0.7377777777777778,
+      1.3733333333333333,
+      1.2927777777777778,
+      0.95,
+      1.0577777777777777,
+      0.37333333333333335,
+      0.8216666666666667,
+      1.115,
+      0.715,
+      0.9677777777777777,
+      1.4255555555555555,
+      1.5383333333333333,
+      0.7711111111111111,
+      0.9172222222222223,
+      1.2066666666666668,
+      1.1044444444444443,
+      1.031111111111111,
+      0.7727777777777778,
+      0.6338888888888888
+    ],
+    "model_probs": [
+      0.00449417890409677,
+      0.0037316398134427213,
+      0.003346809508001343,
+      0.00284237410970519,
+      0.0025965919452285153,
+      0.003384134191733968
+    ],
+    "model_info_gains": [
+      -0.7404358694232646,
+      -0.15536763417059993,
+      -0.03516182605093057,
+      -0.05585686176108763,
+      0.2819978314515805
+    ],
+    "model_delta_probs": [
+      -0.0007625390906540493,
+      -0.0003848303054413786,
+      -0.0005044353982961525,
+      -0.0002457821644766748,
+      0.0007875422465054527
+    ],
+    "model_expected_reward_accs": [
+      0.5233333333333333,
+      0.42333333333333334,
+      0.43333333333333335,
+      0.4166666666666667,
+      0.43333333333333335
+    ],
+    "model_expected_consistencies": [
+      0.3,
+      0.7366666666666667,
+      0.7466666666666667,
+      0.7466666666666667,
+      0.76
+    ],
+    "model_expected_l2s": [
+      0.7100164898388748,
+      1.1617579956946913,
+      1.1293918492361876,
+      1.1548122308022508,
+      1.1315711151709373,
+      1.0792239242074162
+    ],
+    "model_expected_random_set_accs": [
+      0.6794666666666672,
+      0.45593333333333363,
+      0.4742666666666668,
+      0.4574333333333334,
+      0.4665666666666663,
+      0.4901333333333332
+    ],
+    "model_expected_random_set_idxs": [
+      0.4122222222222222,
+      1.1944444444444444,
+      1.4022222222222223,
+      0.7955555555555556,
+      0.7844444444444445,
+      1.295,
+      1.2627777777777778,
+      0.7555555555555555,
+      1.478888888888889,
+      1.426111111111111,
+      0.6888888888888889,
+      1.1744444444444444,
+      0.9633333333333334,
+      0.7777777777777778,
+      1.0544444444444445,
+      1.2738888888888888,
+      0.9566666666666667,
+      1.16,
+      0.6333333333333333,
+      0.8061111111111111,
+      1.2538888888888888,
+      0.8183333333333334,
+      0.6266666666666667,
+      0.8922222222222222,
+      1.1127777777777779,
+      1.1111111111111112,
+      0.8838888888888888,
+      1.461111111111111,
+      0.8855555555555555,
+      0.7027777777777777,
+      1.0305555555555554,
+      0.6827777777777778,
+      1.0716666666666668,
+      0.84,
+      1.125,
+      0.6144444444444445,
+      0.4855555555555556,
+      0.5088888888888888,
+      1.2055555555555555,
+      1.0677777777777777,
+      1.1827777777777777,
+      0.5955555555555555,
+      0.6172222222222222,
+      0.7005555555555556,
+      0.6155555555555555,
+      0.33444444444444443,
+      0.8761111111111111,
+      1.2838888888888889,
+      1.4855555555555555,
+      1.2516666666666667,
+      0.5411111111111111,
+      1.0727777777777778,
+      1.4677777777777778,
+      1.0644444444444445,
+      0.4127777777777778,
+      1.2377777777777779,
+      0.39222222222222225,
+      0.7611111111111111,
+      0.7405555555555555,
+      1.08,
+      0.7855555555555556,
+      0.9905555555555555,
+      1.2838888888888889,
+      1.048888888888889,
+      1.1455555555555557,
+      1.021111111111111,
+      1.031111111111111,
+      0.4411111111111111,
+      1.4866666666666666,
+      1.548888888888889,
+      0.40555555555555556,
+      1.231111111111111,
+      1.2227777777777777,
+      0.9566666666666667,
+      1.2083333333333333,
+      1.1072222222222223,
+      1.248888888888889,
+      1.3066666666666666,
+      0.9972222222222222,
+      1.0161111111111112,
+      1.0327777777777778,
+      0.6694444444444444,
+      1.4094444444444445,
+      1.3022222222222222,
+      0.8994444444444445,
+      0.9766666666666667,
+      0.3638888888888889,
+      0.8766666666666667,
+      1.1294444444444445,
+      0.5661111111111111,
+      0.7983333333333333,
+      1.4494444444444445,
+      1.5527777777777778,
+      0.7427777777777778,
+      0.8616666666666667,
+      1.1094444444444445,
+      1.1344444444444444,
+      1.0977777777777777,
+      0.79,
+      0.5916666666666667
+    ],
+    "infer_reward_accs": [
+      0.5833333333333334,
+      0.5666666666666667,
+      0.5266666666666666,
+      0.5666666666666667,
+      0.58
+    ],
+    "infer_l2s": [
+      0.6798868004826828,
+      0.7377420081799587,
+      0.8204232348268476,
+      0.864092156262234,
+      0.8933916408010356,
+      0.9356221417366755
+    ],
+    "infer_random_set_accs": [
+      0.6862999999999999,
+      0.6619666666666665,
+      0.6238666666666663,
+      0.5970999999999994,
+      0.584866666666666,
+      0.5599333333333333
+    ],
+    "infer_random_set_idxs": [
+      0.275,
+      1.0094444444444444,
+      1.676111111111111,
+      0.895,
+      0.8611111111111112,
+      1.6277777777777778,
+      1.3377777777777777,
+      0.79,
+      1.5427777777777778,
+      1.3405555555555555,
+      0.41888888888888887,
+      1.5522222222222222,
+      1.0594444444444444,
+      0.3283333333333333,
+      0.7266666666666667,
+      1.2005555555555556,
+      1.0644444444444445,
+      1.218888888888889,
+      0.35333333333333333,
+      0.4811111111111111,
+      1.5844444444444445,
+      0.7133333333333334,
+      0.2727777777777778,
+      0.665,
+      0.8988888888888888,
+      1.2861111111111112,
+      0.5305555555555556,
+      1.3872222222222221,
+      0.9861111111111112,
+      0.45166666666666666,
+      1.0616666666666668,
+      0.2638888888888889,
+      1.2072222222222222,
+      0.3288888888888889,
+      1.0805555555555555,
+      0.215,
+      0.26055555555555554,
+      0.22055555555555556,
+      1.0422222222222222,
+      1.0461111111111112,
+      1.3172222222222223,
+      0.29388888888888887,
+      0.4027777777777778,
+      0.4777777777777778,
+      0.23833333333333334,
+      0.23444444444444446,
+      0.9927777777777778,
+      1.075,
+      1.683888888888889,
+      1.173888888888889,
+      0.16555555555555557,
+      0.9588888888888889,
+      1.8294444444444444,
+      1.0822222222222222,
+      0.2361111111111111,
+      1.2833333333333334,
+      0.15555555555555556,
+      0.8766666666666667,
+      0.7611111111111111,
+      1.0261111111111112,
+      0.4583333333333333,
+      0.5261111111111111,
+      1.1,
+      0.9377777777777778,
+      1.0366666666666666,
+      0.8477777777777777,
+      0.9855555555555555,
+      0.8344444444444444,
+      1.7505555555555556,
+      1.7261111111111112,
+      0.2638888888888889,
+      1.4388888888888889,
+      1.6366666666666667,
+      1.145,
+      1.5766666666666667,
+      1.4072222222222222,
+      1.505,
+      1.4105555555555556,
+      0.9866666666666667,
+      0.8744444444444445,
+      1.2455555555555555,
+      0.3616666666666667,
+      1.643888888888889,
+      1.1194444444444445,
+      1.1466666666666667,
+      1.2094444444444445,
+      0.21833333333333332,
+      0.9788888888888889,
+      1.3266666666666667,
+      0.8061111111111111,
+      1.0727777777777778,
+      1.5783333333333334,
+      1.853888888888889,
+      0.4388888888888889,
+      0.5316666666666666,
+      1.055,
+      1.106111111111111,
+      1.3772222222222221,
+      0.7261111111111112,
+      0.4961111111111111
+    ],
+    "infer_probs": [
+      0.00449417890409677,
+      0.006337654465435966,
+      0.0068722604011175765,
+      0.006571771176531911,
+      0.009971259358571842,
+      0.00877979475306347
+    ],
+    "infer_info_gains": [
+      -2.108109296212909,
+      -1.9066600232713422,
+      -0.9040342620114579,
+      -0.7467240459077457,
+      -0.5495108978546918
+    ],
+    "infer_delta_probs": [
+      0.0018434755613391948,
+      0.0005346059356816112,
+      -0.0003004892245856657,
+      0.0033994881820399313,
+      -0.0011914646055083721
+    ],
+    "infer_expected_reward_accs": [
+      0.5233333333333333,
+      0.6,
+      0.6066666666666667,
+      0.5933333333333334,
+      0.5833333333333334
+    ],
+    "infer_expected_l2s": [
+      0.7100164898388748,
+      0.8609748703876052,
+      0.8623329754054614,
+      0.8435167361685524,
+      0.8384577623406639,
+      0.8350020311090884
+    ],
+    "infer_expected_random_set_accs": [
+      0.6794666666666672,
+      0.6049333333333334,
+      0.6043666666666662,
+      0.6010666666666664,
+      0.5805666666666667,
+      0.5558333333333333
+    ],
+    "infer_expected_random_set_idxs": [
+      0.38055555555555554,
+      0.9844444444444445,
+      1.5505555555555555,
+      0.8177777777777778,
+      0.7566666666666667,
+      1.613888888888889,
+      1.2994444444444444,
+      0.7233333333333334,
+      1.5638888888888889,
+      1.4961111111111112,
+      0.6033333333333334,
+      1.316111111111111,
+      0.9833333333333333,
+      0.5033333333333333,
+      0.9627777777777777,
+      1.1816666666666666,
+      0.9738888888888889,
+      1.346111111111111,
+      0.47055555555555556,
+      0.685,
+      1.3911111111111112,
+      0.655,
+      0.34555555555555556,
+      0.8172222222222222,
+      0.9038888888888889,
+      1.238888888888889,
+      0.6133333333333333,
+      1.3611111111111112,
+      0.9922222222222222,
+      0.5183333333333333,
+      1.0994444444444444,
+      0.34833333333333333,
+      1.0905555555555555,
+      0.44333333333333336,
+      1.09,
+      0.33166666666666667,
+      0.6616666666666666,
+      0.2872222222222222,
+      1.0788888888888888,
+      1.0122222222222221,
+      1.175,
+      0.25666666666666665,
+      0.41888888888888887,
+      0.5805555555555556,
+      0.2288888888888889,
+      0.2688888888888889,
+      1.0322222222222222,
+      1.0444444444444445,
+      1.6022222222222222,
+      1.1933333333333334,
+      0.17888888888888888,
+      0.9494444444444444,
+      1.7938888888888889,
+      1.113888888888889,
+      0.3111111111111111,
+      1.0255555555555556,
+      0.20722222222222222,
+      0.8761111111111111,
+      0.7822222222222223,
+      1.028888888888889,
+      0.515,
+      0.9766666666666667,
+      1.1827777777777777,
+      0.8972222222222223,
+      1.0027777777777778,
+      0.8855555555555555,
+      1.0,
+      0.6322222222222222,
+      1.7522222222222221,
+      1.7016666666666667,
+      0.3422222222222222,
+      1.3666666666666667,
+      1.4022222222222223,
+      1.176111111111111,
+      1.4222222222222223,
+      1.2872222222222223,
+      1.393888888888889,
+      1.2538888888888888,
+      0.9616666666666667,
+      0.8244444444444444,
+      1.0972222222222223,
+      0.3516666666666667,
+      1.5644444444444445,
+      1.1888888888888889,
+      1.1533333333333333,
+      0.9027777777777778,
+      0.21722222222222223,
+      1.0877777777777777,
+      1.2066666666666668,
+      0.5355555555555556,
+      0.8072222222222222,
+      1.5766666666666667,
+      1.791111111111111,
+      0.5277777777777778,
+      0.6427777777777778,
+      1.048888888888889,
+      1.1588888888888889,
+      1.39,
+      0.715,
+      0.5744444444444444
+    ],
+    "kls": [
+      0.0,
+      1.5064255137104658,
+      2.5535641034544088,
+      3.3140321220999613,
+      3.5194670098847216,
+      3.4394883157087066
+    ],
+    "model_random_set_direct_accs": [
+      0.39120000000000016,
+      0.4256333333333331,
+      0.4351000000000001,
+      0.4396,
+      0.4506000000000002,
+      0.45826666666666666
+    ],
+    "model_random_set_direct_idxs": [
+      0.8538888888888889,
+      1.2016666666666667,
+      1.3744444444444444,
+      1.103888888888889,
+      0.8572222222222222,
+      0.7166666666666667,
+      0.8122222222222222,
+      0.6766666666666666,
+      1.2627777777777778,
+      1.656111111111111,
+      1.0322222222222222,
+      0.8494444444444444,
+      1.191111111111111,
+      0.7977777777777778,
+      1.4961111111111112,
+      1.4833333333333334,
+      1.1505555555555556,
+      1.255,
+      0.7305555555555555,
+      1.0122222222222221,
+      0.7655555555555555,
+      0.9183333333333333,
+      0.7877777777777778,
+      1.376111111111111,
+      1.0061111111111112,
+      1.0416666666666667,
+      0.86,
+      1.5077777777777779,
+      0.7844444444444445,
+      1.0427777777777778,
+      1.2638888888888888,
+      1.1511111111111112,
+      0.3927777777777778,
+      1.2238888888888888,
+      1.018888888888889,
+      0.8338888888888889,
+      0.7961111111111111,
+      0.5972222222222222,
+      1.2827777777777778,
+      0.6794444444444444,
+      1.3027777777777778,
+      1.0072222222222222,
+      0.6516666666666666,
+      0.9988888888888889,
+      0.8166666666666667,
+      0.4072222222222222,
+      0.8922222222222222,
+      1.39,
+      1.29,
+      1.1527777777777777,
+      0.8861111111111111,
+      0.8311111111111111,
+      1.4755555555555555,
+      1.315,
+      0.97,
+      1.1966666666666668,
+      0.64,
+      0.72,
+      1.2361111111111112,
+      0.9705555555555555,
+      0.9155555555555556,
+      0.6855555555555556,
+      1.24,
+      0.5738888888888889,
+      1.2066666666666668,
+      1.0227777777777778,
+      0.99,
+      0.4722222222222222,
+      1.1527777777777777,
+      1.1294444444444445,
+      0.76,
+      0.7577777777777778,
+      1.2383333333333333,
+      0.6222222222222222,
+      1.1944444444444444,
+      0.7,
+      0.9672222222222222,
+      1.0294444444444444,
+      0.9555555555555556,
+      1.0972222222222223,
+      0.9783333333333334,
+      0.6888888888888889,
+      1.0372222222222223,
+      1.136111111111111,
+      1.1527777777777777,
+      0.9366666666666666,
+      0.47444444444444445,
+      0.7672222222222222,
+      0.96,
+      1.0477777777777777,
+      0.6144444444444445,
+      1.2544444444444445,
+      1.5216666666666667,
+      1.0216666666666667,
+      1.2655555555555555,
+      0.7422222222222222,
+      1.1183333333333334,
+      0.49944444444444447,
+      0.45555555555555555,
+      0.6966666666666667
+    ],
+    "model_expected_random_set_consistencies": [
+      0.3400000000000023,
+      0.7262666666666673,
+      0.7321000000000001,
+      0.7591333333333339,
+      0.7480333333333337,
+      0.7335666666666668
+    ],
+    "model_random_set_consistencies": [
+      0.3564666666666662,
+      0.6871666666666677,
+      0.6703333333333337,
+      0.6981,
+      0.7074333333333329,
+      0.6896333333333329
+    ]
+  }
+}

results_v2/human_v2_5rounds_seed1/metrics.json

@@ -0,0 +1,769 @@
+{
+  "config": {
+    "task_name": "flight",
+    "prior": "lm",
+    "generator_name": "random",
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "features": [
+      "departure_time",
+      "duration",
+      "number_of_stops",
+      "price"
+    ],
+    "reward_choices": {
+      "departure_time": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "duration": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "number_of_stops": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ],
+      "price": [
+        -1,
+        -0.5,
+        0,
+        0.5,
+        1
+      ]
+    },
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "text",
+    "reward_type": "scoring",
+    "max_rounds": 5,
+    "num_options": 3,
+    "num_candidates": 1000,
+    "eval_mode": "conversation",
+    "use_cot": false,
+    "num_random_set_examples": 100,
+    "seed": 0,
+    "debug": false,
+    "data_file": "/data/cl/user/linluqiu/bayesian_inference/human_as_human_v2/human_v2_5rounds_seed1.jsonl",
+    "random_set_file": "data/eval_data/flight_random_set_100.json",
+    "prior_file": null,
+    "eval_every": 1,
+    "likelihood_smoothing": 0.0
+  },
+  "metrics": {
+    "accs": [
+      0.30333333333333334,
+      0.44,
+      0.5133333333333333,
+      0.45,
+      0.44333333333333336
+    ],
+    "model_accs": [
+      0.30333333333333334,
+      0.44,
+      0.5133333333333333,
+      0.45,
+      0.44333333333333336
+    ],
+    "model_reward_accs": [
+      0.6033333333333334,
+      0.4166666666666667,
+      0.49333333333333335,
+      0.5066666666666667,
+      0.42
+    ],
+    "model_consistencies": [
+      0.39,
+      0.7033333333333334,
+      0.7133333333333334,
+      0.74,
+      0.6933333333333334
+    ],
+    "model_l2s": [
+      0.6798868004826828,
+      1.1506890933944822,
+      1.151075757942749,
+      1.104910538930452,
+      1.1050879210258626,
+      1.034734988728362
+    ],
+    "model_random_set_accs": [
+      0.6864666666666671,
+      0.46123333333333333,
+      0.45816666666666656,
+      0.4826666666666667,
+      0.48663333333333364,
+      0.520333333333333
+    ],
+    "model_random_set_idxs": [
+      0.4038888888888889,
+      1.1294444444444445,
+      1.5294444444444444,
+      0.9294444444444444,
+      0.8511111111111112,
+      1.3155555555555556,
+      1.1444444444444444,
+      0.8522222222222222,
+      1.385,
+      1.3322222222222222,
+      0.5211111111111111,
+      1.1294444444444445,
+      1.0955555555555556,
+      0.6377777777777778,
+      0.8883333333333333,
+      1.2216666666666667,
+      1.0044444444444445,
+      1.0911111111111111,
+      0.6077777777777778,
+      0.6177777777777778,
+      1.2327777777777778,
+      0.7872222222222223,
+      0.5777777777777777,
+      0.8016666666666666,
+      0.9816666666666667,
+      1.1861111111111111,
+      0.8172222222222222,
+      1.3622222222222222,
+      0.8927777777777778,
+      0.6988888888888889,
+      0.99,
+      0.6655555555555556,
+      1.0483333333333333,
+      0.7194444444444444,
+      1.1183333333333334,
+      0.4961111111111111,
+      0.27055555555555555,
+      0.5538888888888889,
+      1.1683333333333332,
+      1.1116666666666666,
+      1.2961111111111112,
+      0.5522222222222222,
+      0.5594444444444444,
+      0.6488888888888888,
+      0.5377777777777778,
+      0.29777777777777775,
+      0.8333333333333334,
+      1.2683333333333333,
+      1.5366666666666666,
+      1.2327777777777778,
+      0.45111111111111113,
+      1.0244444444444445,
+      1.5855555555555556,
+      1.0372222222222223,
+      0.43666666666666665,
+      1.3094444444444444,
+      0.3511111111111111,
+      0.7988888888888889,
+      0.8572222222222222,
+      1.025,
+      0.7533333333333333,
+      0.7344444444444445,
+      1.2022222222222223,
+      1.0372222222222223,
+      1.145,
+      1.0744444444444445,
+      1.0338888888888889,
+      0.6327777777777778,
+      1.4922222222222221,
+      1.5161111111111112,
+      0.4011111111111111,
+      1.1272222222222221,
+      1.3594444444444445,
+      0.9594444444444444,
+      1.3683333333333334,
+      1.1527777777777777,
+      1.3033333333333332,
+      1.2338888888888888,
+      1.01,
+      1.0805555555555555,
+      1.0955555555555556,
+      0.6655555555555556,
+      1.4333333333333333,
+      1.2538888888888888,
+      0.9794444444444445,
+      1.0794444444444444,
+      0.35333333333333333,
+      0.8855555555555555,
+      1.1383333333333334,
+      0.7561111111111111,
+      0.945,
+      1.4361111111111111,
+      1.6411111111111112,
+      0.6961111111111111,
+      0.78,
+      1.1677777777777778,
+      1.0472222222222223,
+      1.1388888888888888,
+      0.7511111111111111,
+      0.6466666666666666
+    ],
+    "model_probs": [
+      0.004477773753145205,
+      0.00418596449828821,
+      0.003718455684560003,
+      0.004237958268559131,
+      0.004870414690876288,
+      0.004388178074071476
+    ],
+    "model_info_gains": [
+      -0.5767115929237672,
+      -0.2072350120111503,
+      -0.08022150081766044,
+      0.004830756032638807,
+      0.23035137439851544
+    ],
+    "model_delta_probs": [
+      -0.0002918092548569954,
+      -0.0004675088137282065,
+      0.0005195025839991274,
+      0.0006324564223171573,
+      -0.000482236616804812
+    ],
+    "model_expected_reward_accs": [
+      0.56,
+      0.44333333333333336,
+      0.5266666666666666,
+      0.5333333333333333,
+      0.49333333333333335
+    ],
+    "model_expected_consistencies": [
+      0.37,
+      0.7566666666666667,
+      0.7466666666666667,
+      0.77,
+      0.7533333333333333
+    ],
+    "model_expected_l2s": [
+      0.7061205420316785,
+      1.1314290818470303,
+      1.1187242604240566,
+      1.0876086940318563,
+      1.0809900310163219,
+      1.0247519391235234
+    ],
+    "model_expected_random_set_accs": [
+      0.6794666666666672,
+      0.4714333333333334,
+      0.4778000000000003,
+      0.4922666666666665,
+      0.49663333333333326,
+      0.5239666666666665
+    ],
+    "model_expected_random_set_idxs": [
+      0.44722222222222224,
+      1.1605555555555556,
+      1.4583333333333333,
+      0.8905555555555555,
+      0.8277777777777777,
+      1.3883333333333334,
+      1.2638888888888888,
+      0.8077777777777778,
+      1.5255555555555556,
+      1.44,
+      0.6561111111111111,
+      1.1994444444444445,
+      1.01,
+      0.6605555555555556,
+      0.9672222222222222,
+      1.2461111111111112,
+      0.9972222222222222,
+      1.1611111111111112,
+      0.5922222222222222,
+      0.7066666666666667,
+      1.3033333333333332,
+      0.8027777777777778,
+      0.5377777777777778,
+      0.8072222222222222,
+      0.9894444444444445,
+      1.14,
+      0.84,
+      1.3705555555555555,
+      0.8938888888888888,
+      0.6755555555555556,
+      1.0172222222222222,
+      0.6083333333333333,
+      1.0772222222222223,
+      0.7705555555555555,
+      1.0777777777777777,
+      0.5527777777777778,
+      0.4461111111111111,
+      0.4961111111111111,
+      1.1866666666666668,
+      1.1044444444444443,
+      1.2277777777777779,
+      0.5311111111111111,
+      0.59,
+      0.6727777777777778,
+      0.5066666666666667,
+      0.31666666666666665,
+      0.8911111111111111,
+      1.2483333333333333,
+      1.5116666666666667,
+      1.2316666666666667,
+      0.44333333333333336,
+      1.0305555555555554,
+      1.5722222222222222,
+      1.0544444444444445,
+      0.37722222222222224,
+      1.2172222222222222,
+      0.37444444444444447,
+      0.7655555555555555,
+      0.7838888888888889,
+      1.065,
+      0.7205555555555555,
+      1.0583333333333333,
+      1.2683333333333333,
+      1.0566666666666666,
+      1.1166666666666667,
+      1.0227777777777778,
+      1.0422222222222222,
+      0.48277777777777775,
+      1.5416666666666667,
+      1.5794444444444444,
+      0.3933333333333333,
+      1.2633333333333334,
+      1.2783333333333333,
+      0.9883333333333333,
+      1.3366666666666667,
+      1.1455555555555557,
+      1.2544444444444445,
+      1.3122222222222222,
+      0.9244444444444444,
+      1.0094444444444444,
+      1.1005555555555555,
+      0.5905555555555555,
+      1.4694444444444446,
+      1.285,
+      0.9355555555555556,
+      0.9694444444444444,
+      0.3472222222222222,
+      0.9244444444444444,
+      1.123888888888889,
+      0.6,
+      0.7505555555555555,
+      1.498888888888889,
+      1.6366666666666667,
+      0.6644444444444444,
+      0.7416666666666667,
+      1.0916666666666666,
+      1.0833333333333333,
+      1.2094444444444445,
+      0.7761111111111111,
+      0.5944444444444444
+    ],
+    "infer_reward_accs": [
+      0.58,
+      0.53,
+      0.5433333333333333,
+      0.56,
+      0.53
+    ],
+    "infer_l2s": [
+      0.6798868004826828,
+      0.7966104702092957,
+      0.8542393358907556,
+      0.8585170330619373,
+      0.8933513521194366,
+      0.938662316869842
+    ],
+    "infer_random_set_accs": [
+      0.6862999999999999,
+      0.6323666666666663,
+      0.6070666666666666,
+      0.5997,
+      0.5835666666666663,
+      0.5589666666666666
+    ],
+    "infer_random_set_idxs": [
+      0.3038888888888889,
+      0.9977777777777778,
+      1.6666666666666667,
+      0.8861111111111111,
+      0.87,
+      1.6283333333333334,
+      1.3277777777777777,
+      0.7955555555555556,
+      1.5305555555555554,
+      1.3466666666666667,
+      0.4116666666666667,
+      1.5927777777777778,
+      1.0733333333333333,
+      0.33444444444444443,
+      0.7333333333333333,
+      1.2044444444444444,
+      1.0572222222222223,
+      1.2283333333333333,
+      0.3877777777777778,
+      0.4938888888888889,
+      1.535,
+      0.7316666666666667,
+      0.2594444444444444,
+      0.6316666666666667,
+      0.8966666666666666,
+      1.2833333333333334,
+      0.5522222222222222,
+      1.3605555555555555,
+      1.001111111111111,
+      0.4438888888888889,
+      1.051111111111111,
+      0.2788888888888889,
+      1.2022222222222223,
+      0.345,
+      1.08,
+      0.21777777777777776,
+      0.275,
+      0.2538888888888889,
+      1.008888888888889,
+      1.0444444444444445,
+      1.3072222222222223,
+      0.2972222222222222,
+      0.38,
+      0.48333333333333334,
+      0.24666666666666667,
+      0.2427777777777778,
+      0.9716666666666667,
+      1.076111111111111,
+      1.6605555555555556,
+      1.1766666666666667,
+      0.17555555555555555,
+      0.9405555555555556,
+      1.8172222222222223,
+      1.0705555555555555,
+      0.2866666666666667,
+      1.2722222222222221,
+      0.17333333333333334,
+      0.8755555555555555,
+      0.7294444444444445,
+      1.018888888888889,
+      0.4911111111111111,
+      0.5261111111111111,
+      1.1077777777777778,
+      0.9388888888888889,
+      1.0338888888888889,
+      0.8394444444444444,
+      0.99,
+      0.8422222222222222,
+      1.7016666666666667,
+      1.7055555555555555,
+      0.2833333333333333,
+      1.4416666666666667,
+      1.6277777777777778,
+      1.1277777777777778,
+      1.575,
+      1.4072222222222222,
+      1.47,
+      1.4377777777777778,
+      0.9866666666666667,
+      0.8883333333333333,
+      1.2733333333333334,
+      0.37444444444444447,
+      1.6488888888888888,
+      1.0977777777777777,
+      1.1205555555555555,
+      1.2005555555555556,
+      0.23,
+      0.9388888888888889,
+      1.36,
+      0.7822222222222223,
+      1.0577777777777777,
+      1.5827777777777778,
+      1.8311111111111111,
+      0.46444444444444444,
+      0.5411111111111111,
+      1.0622222222222222,
+      1.0905555555555555,
+      1.396111111111111,
+      0.7244444444444444,
+      0.4766666666666667
+    ],
+    "infer_probs": [
+      0.004477773753145205,
+      0.006072530801563213,
+      0.0049616723118621545,
+      0.004807750138376529,
+      0.0091766605937543,
+      0.008711555347933124
+    ],
+    "infer_info_gains": [
+      -2.133950571345106,
+      -1.945856628662956,
+      -0.8804176461352968,
+      -0.6928890644942192,
+      -0.583824447602663
+    ],
+    "infer_delta_probs": [
+      0.0015947570484180081,
+      -0.0011108584897010587,
+      -0.00015392217348562555,
+      0.004368910455377772,
+      -0.000465105245821178
+    ],
+    "infer_expected_reward_accs": [
+      0.56,
+      0.5333333333333333,
+      0.5666666666666667,
+      0.6166666666666667,
+      0.5933333333333334
+    ],
+    "infer_expected_l2s": [
+      0.7061205420316785,
+      0.8769894483490475,
+      0.8796962928556044,
+      0.8509228634884637,
+      0.8348700042332291,
+      0.8349702751337066
+    ],
+    "infer_expected_random_set_accs": [
+      0.6794666666666672,
+      0.6035666666666665,
+      0.5922666666666672,
+      0.5945333333333336,
+      0.5867333333333337,
+      0.5559666666666665
+    ],
+    "infer_expected_random_set_idxs": [
+      0.405,
+      0.9683333333333334,
+      1.5677777777777777,
+      0.8266666666666667,
+      0.7772222222222223,
+      1.6288888888888888,
+      1.2905555555555555,
+      0.7455555555555555,
+      1.57,
+      1.485,
+      0.5777777777777777,
+      1.343888888888889,
+      0.9838888888888889,
+      0.5061111111111111,
+      0.9372222222222222,
+      1.1694444444444445,
+      0.9688888888888889,
+      1.33,
+      0.49944444444444447,
+      0.6733333333333333,
+      1.3416666666666666,
+      0.6722222222222223,
+      0.3438888888888889,
+      0.7655555555555555,
+      0.8944444444444445,
+      1.2405555555555556,
+      0.6327777777777778,
+      1.3483333333333334,
+      0.9994444444444445,
+      0.4961111111111111,
+      1.1077777777777778,
+      0.36444444444444446,
+      1.1188888888888888,
+      0.45111111111111113,
+      1.0827777777777778,
+      0.36444444444444446,
+      0.6266666666666667,
+      0.32,
+      1.0438888888888889,
+      1.0327777777777778,
+      1.1866666666666668,
+      0.2677777777777778,
+      0.4172222222222222,
+      0.5922222222222222,
+      0.24055555555555555,
+      0.2738888888888889,
+      1.0333333333333334,
+      1.0605555555555555,
+      1.5922222222222222,
+      1.1861111111111111,
+      0.19555555555555557,
+      0.9388888888888889,
+      1.785,
+      1.0944444444444446,
+      0.3466666666666667,
+      1.0355555555555556,
+      0.20333333333333334,
+      0.8738888888888889,
+      0.7427777777777778,
+      1.0338888888888889,
+      0.5477777777777778,
+      0.99,
+      1.1883333333333332,
+      0.9111111111111111,
+      1.0083333333333333,
+      0.8877777777777778,
+      1.0122222222222221,
+      0.66,
+      1.7405555555555556,
+      1.7272222222222222,
+      0.365,
+      1.356111111111111,
+      1.44,
+      1.1744444444444444,
+      1.4583333333333333,
+      1.281111111111111,
+      1.3772222222222221,
+      1.2883333333333333,
+      0.95,
+      0.7966666666666666,
+      1.1194444444444445,
+      0.3422222222222222,
+      1.5888888888888888,
+      1.195,
+      1.1516666666666666,
+      0.9366666666666666,
+      0.21055555555555555,
+      1.0577777777777777,
+      1.2122222222222223,
+      0.5344444444444445,
+      0.7933333333333333,
+      1.586111111111111,
+      1.7761111111111112,
+      0.49333333333333335,
+      0.6222222222222222,
+      1.0555555555555556,
+      1.1566666666666667,
+      1.3983333333333334,
+      0.6888888888888889,
+      0.5577777777777778
+    ],
+    "kls": [
+      0.0,
+      1.4903185888200916,
+      2.5070598054996287,
+      3.2354705382165356,
+      3.5226605453238586,
+      3.3193599388646233
+    ],
+    "model_random_set_direct_accs": [
+      0.3984333333333337,
+      0.4372999999999998,
+      0.44676666666666676,
+      0.463766666666666,
+      0.47029999999999994,
+      0.4858000000000003
+    ],
+    "model_random_set_direct_idxs": [
+      0.8372222222222222,
+      1.1272222222222221,
+      1.481111111111111,
+      1.145,
+      0.8683333333333333,
+      0.7572222222222222,
+      0.8188888888888889,
+      0.715,
+      1.2377777777777779,
+      1.6322222222222222,
+      1.1283333333333334,
+      0.915,
+      1.18,
+      0.7827777777777778,
+      1.4594444444444445,
+      1.468888888888889,
+      1.14,
+      1.221111111111111,
+      0.715,
+      0.9194444444444444,
+      0.7233333333333334,
+      0.8827777777777778,
+      0.7494444444444445,
+      1.25,
+      0.91,
+      1.0316666666666667,
+      0.9916666666666667,
+      1.4761111111111112,
+      0.8427777777777777,
+      0.9788888888888889,
+      1.2733333333333334,
+      1.0394444444444444,
+      0.45166666666666666,
+      1.193888888888889,
+      0.9877777777777778,
+      0.8038888888888889,
+      0.765,
+      0.6205555555555555,
+      1.041111111111111,
+      0.6861111111111111,
+      1.3411111111111111,
+      0.9555555555555556,
+      0.6561111111111111,
+      0.9005555555555556,
+      0.7544444444444445,
+      0.3877777777777778,
+      0.9138888888888889,
+      1.3211111111111111,
+      1.3122222222222222,
+      1.135,
+      0.815,
+      0.8038888888888889,
+      1.5105555555555557,
+      1.2705555555555557,
+      0.925,
+      1.1944444444444444,
+      0.5711111111111111,
+      0.7477777777777778,
+      1.268888888888889,
+      1.0116666666666667,
+      0.8927777777777778,
+      0.6883333333333334,
+      1.1827777777777777,
+      0.6077777777777778,
+      1.1772222222222222,
+      0.9894444444444445,
+      1.0016666666666667,
+      0.515,
+      1.1355555555555557,
+      1.4883333333333333,
+      0.8088888888888889,
+      0.7577777777777778,
+      1.2927777777777778,
+      0.8727777777777778,
+      1.2677777777777777,
+      0.7572222222222222,
+      0.9111111111111111,
+      1.0361111111111112,
+      0.9194444444444444,
+      1.1216666666666666,
+      1.0722222222222222,
+      0.6633333333333333,
+      1.0583333333333333,
+      1.1344444444444444,
+      1.2138888888888888,
+      0.9244444444444444,
+      0.3888888888888889,
+      0.8405555555555555,
+      1.0305555555555554,
+      1.0627777777777778,
+      0.5666666666666667,
+      1.2477777777777779,
+      1.6155555555555556,
+      0.9805555555555555,
+      1.1972222222222222,
+      0.7411111111111112,
+      1.1188888888888888,
+      0.5372222222222223,
+      0.44222222222222224,
+      0.7027777777777777
+    ],
+    "model_expected_random_set_consistencies": [
+      0.35999999999999954,
+      0.7240333333333331,
+      0.7432666666666674,
+      0.7603666666666669,
+      0.7648000000000001,
+      0.7585000000000001
+    ],
+    "model_random_set_consistencies": [
+      0.37646666666666656,
+      0.671766666666667,
+      0.6909000000000003,
+      0.7138333333333338,
+      0.7259333333333334,
+      0.7143333333333332
+    ]
+  }
+}

results_v2/webshop_bedframes_format_small/metrics.json

@@ -0,0 +1,143 @@
+{
+  "config": {
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "task_name": "webshop",
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "name",
+    "max_rounds": 5,
+    "num_options": 3,
+    "eval_mode": "conversation",
+    "num_random_set_examples": 100,
+    "debug": false,
+    "data_file": "/data/cl/user/linluqiu/bayesian_inference/webshop_eval_data//bedframes.jsonl",
+    "eval_every": 1
+  },
+  "metrics": {
+    "accs": [
+      0.1,
+      0.5,
+      0.6,
+      0.6,
+      0.7
+    ],
+    "model_random_set_direct_accs": [
+      0.356,
+      0.404,
+      0.471,
+      0.489,
+      0.49400000000000005,
+      0.493
+    ],
+    "model_random_set_direct_idxs": [
+      0.7,
+      1.3333333333333333,
+      0.9166666666666666,
+      1.0833333333333333,
+      0.9833333333333333,
+      0.9166666666666666,
+      1.0833333333333333,
+      1.2166666666666666,
+      1.0166666666666666,
+      1.0833333333333333,
+      1.0333333333333334,
+      0.85,
+      1.1333333333333333,
+      1.2,
+      1.05,
+      1.05,
+      1.1,
+      1.2166666666666666,
+      1.2,
+      1.2666666666666666,
+      1.2166666666666666,
+      0.9333333333333333,
+      1.45,
+      1.4,
+      1.4,
+      1.1666666666666667,
+      1.1166666666666667,
+      1.1666666666666667,
+      1.0333333333333334,
+      1.0833333333333333,
+      1.2333333333333334,
+      1.1666666666666667,
+      1.2833333333333334,
+      1.2833333333333334,
+      1.2833333333333334,
+      1.25,
+      1.1666666666666667,
+      1.2166666666666666,
+      1.2333333333333334,
+      1.0,
+      1.3166666666666667,
+      1.15,
+      0.75,
+      1.2333333333333334,
+      1.3,
+      1.2833333333333334,
+      1.35,
+      0.8666666666666667,
+      1.2,
+      1.0666666666666667,
+      1.45,
+      1.2,
+      1.25,
+      0.9833333333333333,
+      1.5,
+      1.0166666666666666,
+      1.2333333333333334,
+      1.1166666666666667,
+      0.7333333333333333,
+      1.1,
+      1.0333333333333334,
+      1.0666666666666667,
+      1.15,
+      0.75,
+      1.0833333333333333,
+      0.85,
+      1.1833333333333333,
+      0.9833333333333333,
+      1.15,
+      1.1333333333333333,
+      0.8833333333333333,
+      1.05,
+      1.3833333333333333,
+      0.9666666666666667,
+      1.2666666666666666,
+      1.2166666666666666,
+      0.5666666666666667,
+      1.4833333333333334,
+      1.2333333333333334,
+      1.4166666666666667,
+      1.05,
+      1.25,
+      1.1,
+      1.55,
+      1.3666666666666667,
+      0.9333333333333333,
+      0.7166666666666667,
+      0.8166666666666667,
+      1.4333333333333333,
+      1.1833333333333333,
+      0.9666666666666667,
+      1.0833333333333333,
+      0.8833333333333333,
+      0.9333333333333333,
+      1.2166666666666666,
+      1.2166666666666666,
+      1.2,
+      1.2,
+      1.1,
+      1.2666666666666666
+    ],
+    "model_reward_accs": [
+      0.1767245650291443,
+      0.17174454778432846,
+      0.19220797196030617,
+      0.25527173802256586,
+      0.28639224395155904,
+      0.27425263077020645
+    ]
+  }
+}

results_v2/webshop_mens_t-shirts_&_tanks_format_small/metrics.json

@@ -0,0 +1,143 @@
+{
+  "config": {
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "task_name": "webshop",
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "name",
+    "max_rounds": 5,
+    "num_options": 3,
+    "eval_mode": "conversation",
+    "num_random_set_examples": 100,
+    "debug": false,
+    "data_file": "/data/cl/user/linluqiu/bayesian_inference/webshop_eval_data//mens_t-shirts_&_tanks.jsonl",
+    "eval_every": 1
+  },
+  "metrics": {
+    "accs": [
+      0.3,
+      0.3,
+      0.8,
+      0.8,
+      0.7
+    ],
+    "model_random_set_direct_accs": [
+      0.388,
+      0.45999999999999996,
+      0.556,
+      0.583,
+      0.6260000000000001,
+      0.634
+    ],
+    "model_random_set_direct_idxs": [
+      1.4666666666666666,
+      1.4166666666666667,
+      0.95,
+      0.65,
+      1.2833333333333334,
+      1.0,
+      1.2666666666666666,
+      0.9,
+      1.35,
+      1.25,
+      0.9166666666666666,
+      1.2666666666666666,
+      1.1666666666666667,
+      1.1333333333333333,
+      1.2333333333333334,
+      1.0166666666666666,
+      0.8833333333333333,
+      0.9333333333333333,
+      0.8666666666666667,
+      1.0666666666666667,
+      1.05,
+      1.2166666666666666,
+      1.2,
+      1.4833333333333334,
+      0.7,
+      0.6666666666666666,
+      0.8666666666666667,
+      1.0666666666666667,
+      0.9166666666666666,
+      0.7166666666666667,
+      1.25,
+      1.2333333333333334,
+      1.0666666666666667,
+      1.1833333333333333,
+      1.0333333333333334,
+      1.15,
+      1.15,
+      1.2333333333333334,
+      1.1333333333333333,
+      1.05,
+      0.7333333333333333,
+      0.7666666666666667,
+      1.0166666666666666,
+      1.3833333333333333,
+      0.75,
+      1.0333333333333334,
+      1.1666666666666667,
+      1.0166666666666666,
+      1.5,
+      0.9166666666666666,
+      0.95,
+      1.1166666666666667,
+      1.0,
+      1.0833333333333333,
+      0.95,
+      1.15,
+      0.5,
+      1.0833333333333333,
+      0.7,
+      1.1166666666666667,
+      0.7833333333333333,
+      0.8,
+      0.95,
+      1.2333333333333334,
+      1.3333333333333333,
+      0.85,
+      1.1666666666666667,
+      1.1666666666666667,
+      1.25,
+      0.4666666666666667,
+      1.1333333333333333,
+      1.0166666666666666,
+      1.0166666666666666,
+      0.9333333333333333,
+      1.3833333333333333,
+      0.9333333333333333,
+      1.2,
+      1.2166666666666666,
+      1.1833333333333333,
+      1.1,
+      1.2,
+      1.1333333333333333,
+      1.0333333333333334,
+      0.9833333333333333,
+      1.2333333333333334,
+      1.25,
+      0.8333333333333334,
+      1.0833333333333333,
+      0.7333333333333333,
+      0.6,
+      1.15,
+      0.7333333333333333,
+      1.5333333333333334,
+      1.15,
+      0.85,
+      1.0833333333333333,
+      0.8,
+      1.0666666666666667,
+      1.1833333333333333,
+      1.35
+    ],
+    "model_reward_accs": [
+      0.11098223812878132,
+      0.13787838742136954,
+      0.12062152102589607,
+      0.18411711491644384,
+      0.19250422418117524,
+      0.23129856139421462
+    ]
+  }
+}

results_v2/webshop_womens_slippers_format_small/metrics.json

@@ -0,0 +1,143 @@
+{
+  "config": {
+    "model_name": "outputs/gemma2-9b-it_prior_uniform_random_flight_text_10seeds_gt",
+    "task_name": "webshop",
+    "assistant_name": "lm",
+    "formatter_name": "gemma",
+    "option_format": "name",
+    "max_rounds": 5,
+    "num_options": 3,
+    "eval_mode": "conversation",
+    "num_random_set_examples": 100,
+    "debug": false,
+    "data_file": "/data/cl/user/linluqiu/bayesian_inference/webshop_eval_data//womens_slippers.jsonl",
+    "eval_every": 1
+  },
+  "metrics": {
+    "accs": [
+      0.2,
+      0.3,
+      0.3,
+      0.2,
+      0.5
+    ],
+    "model_random_set_direct_accs": [
+      0.356,
+      0.429,
+      0.4069999999999999,
+      0.417,
+      0.44799999999999995,
+      0.504
+    ],
+    "model_random_set_direct_idxs": [
+      1.4333333333333333,
+      1.2333333333333334,
+      1.4,
+      1.1333333333333333,
+      1.0166666666666666,
+      1.1666666666666667,
+      1.4,
+      1.05,
+      0.9833333333333333,
+      1.1666666666666667,
+      0.9,
+      1.0666666666666667,
+      1.15,
+      1.0333333333333334,
+      0.9333333333333333,
+      1.2,
+      1.2166666666666666,
+      1.0333333333333334,
+      1.2166666666666666,
+      1.0666666666666667,
+      1.3,
+      1.3333333333333333,
+      0.9,
+      1.2833333333333334,
+      0.9833333333333333,
+      1.1166666666666667,
+      1.1333333333333333,
+      1.05,
+      1.4166666666666667,
+      1.5,
+      0.95,
+      0.85,
+      0.9666666666666667,
+      0.95,
+      1.1333333333333333,
+      1.15,
+      1.3833333333333333,
+      0.7,
+      1.1666666666666667,
+      1.1333333333333333,
+      0.9833333333333333,
+      1.0166666666666666,
+      1.2,
+      0.9166666666666666,
+      0.9666666666666667,
+      0.9,
+      0.8,
+      1.0333333333333334,
+      0.9166666666666666,
+      1.15,
+      1.15,
+      1.5666666666666667,
+      1.4333333333333333,
+      1.0333333333333334,
+      1.1833333333333333,
+      1.1333333333333333,
+      1.1333333333333333,
+      1.2833333333333334,
+      0.75,
+      1.2833333333333334,
+      1.0833333333333333,
+      0.9333333333333333,
+      1.2666666666666666,
+      1.2333333333333334,
+      1.0,
+      1.35,
+      0.9833333333333333,
+      1.05,
+      0.9833333333333333,
+      1.5833333333333333,
+      1.1,
+      1.1,
+      1.2666666666666666,
+      1.2833333333333334,
+      1.1,
+      0.7833333333333333,
+      1.3666666666666667,
+      1.2166666666666666,
+      0.9166666666666666,
+      1.1,
+      1.1166666666666667,
+      1.1833333333333333,
+      0.9833333333333333,
+      1.1166666666666667,
+      0.6666666666666666,
+      1.0166666666666666,
+      1.1333333333333333,
+      0.95,
+      1.3333333333333333,
+      1.3666666666666667,
+      0.65,
+      1.3,
+      0.9666666666666667,
+      1.0833333333333333,
+      1.3,
+      1.0666666666666667,
+      1.15,
+      0.9833333333333333,
+      0.9666666666666667,
+      1.2333333333333334
+    ],
+    "model_reward_accs": [
+      0.12477145716547966,
+      0.1468495361506939,
+      0.1850356325507164,
+      0.15764938220381736,
+      0.16048554331064224,
+      0.13894478157162665
+    ]
+  }
+}

special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
+size 4241003

tokenizer_config.json

@@ -0,0 +1,2014 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<bos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "5": {
+      "content": "<2mass>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "6": {
+      "content": "[@BOS@]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "7": {
+      "content": "<unused0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<unused1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<unused2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<unused3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<unused4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<unused5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<unused6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<unused7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "15": {
+      "content": "<unused8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "16": {
+      "content": "<unused9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "17": {
+      "content": "<unused10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "18": {
+      "content": "<unused11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "19": {
+      "content": "<unused12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "20": {
+      "content": "<unused13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "21": {
+      "content": "<unused14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "22": {
+      "content": "<unused15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "23": {
+      "content": "<unused16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "24": {
+      "content": "<unused17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "25": {
+      "content": "<unused18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "26": {
+      "content": "<unused19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27": {
+      "content": "<unused20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "28": {
+      "content": "<unused21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "29": {
+      "content": "<unused22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "30": {
+      "content": "<unused23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "31": {
+      "content": "<unused24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32": {
+      "content": "<unused25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "33": {
+      "content": "<unused26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "34": {
+      "content": "<unused27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "35": {
+      "content": "<unused28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "36": {
+      "content": "<unused29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "37": {
+      "content": "<unused30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "38": {
+      "content": "<unused31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "39": {
+      "content": "<unused32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "40": {
+      "content": "<unused33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "41": {
+      "content": "<unused34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "42": {
+      "content": "<unused35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "43": {
+      "content": "<unused36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "44": {
+      "content": "<unused37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "45": {
+      "content": "<unused38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "46": {
+      "content": "<unused39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "47": {
+      "content": "<unused40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "48": {
+      "content": "<unused41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "49": {
+      "content": "<unused42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50": {
+      "content": "<unused43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51": {
+      "content": "<unused44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "52": {
+      "content": "<unused45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "53": {
+      "content": "<unused46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "54": {
+      "content": "<unused47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "55": {
+      "content": "<unused48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "56": {
+      "content": "<unused49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "57": {
+      "content": "<unused50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "58": {
+      "content": "<unused51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "59": {
+      "content": "<unused52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "60": {
+      "content": "<unused53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "61": {
+      "content": "<unused54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "62": {
+      "content": "<unused55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "63": {
+      "content": "<unused56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64": {
+      "content": "<unused57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "65": {
+      "content": "<unused58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "66": {
+      "content": "<unused59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "67": {
+      "content": "<unused60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "68": {
+      "content": "<unused61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "69": {
+      "content": "<unused62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "70": {
+      "content": "<unused63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "71": {
+      "content": "<unused64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "72": {
+      "content": "<unused65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "73": {
+      "content": "<unused66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "74": {
+      "content": "<unused67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "75": {
+      "content": "<unused68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "76": {
+      "content": "<unused69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "77": {
+      "content": "<unused70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "78": {
+      "content": "<unused71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "79": {
+      "content": "<unused72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "80": {
+      "content": "<unused73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81": {
+      "content": "<unused74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "82": {
+      "content": "<unused75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "83": {
+      "content": "<unused76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "84": {
+      "content": "<unused77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "85": {
+      "content": "<unused78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "86": {
+      "content": "<unused79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "87": {
+      "content": "<unused80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "<unused81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "<unused82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "90": {
+      "content": "<unused83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "91": {
+      "content": "<unused84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "92": {
+      "content": "<unused85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "93": {
+      "content": "<unused86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "94": {
+      "content": "<unused87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "95": {
+      "content": "<unused88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "96": {
+      "content": "<unused89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "97": {
+      "content": "<unused90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "98": {
+      "content": "<unused91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "99": {
+      "content": "<unused92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100": {
+      "content": "<unused93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "101": {
+      "content": "<unused94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102": {
+      "content": "<unused95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "103": {
+      "content": "<unused96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "104": {
+      "content": "<unused97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "105": {
+      "content": "<unused98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "106": {
+      "content": "<start_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "107": {
+      "content": "<end_of_turn>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "108": {
+      "content": "\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "109": {
+      "content": "\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
+      "content": "\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "111": {
+      "content": "\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "112": {
+      "content": "\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "113": {
+      "content": "\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "114": {
+      "content": "\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "115": {
+      "content": "\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "116": {
+      "content": "\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "117": {
+      "content": "\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "118": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "119": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "120": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "121": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "122": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "123": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "124": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "125": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "126": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "127": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "129": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "130": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "131": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "132": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "133": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "134": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "135": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "136": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "137": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "138": {
+      "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "139": {
+      "content": "▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "140": {
+      "content": "▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "141": {
+      "content": "▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "142": {
+      "content": "▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "143": {
+      "content": "▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "144": {
+      "content": "▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "145": {
+      "content": "▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "146": {
+      "content": "▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "147": {
+      "content": "▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "148": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "149": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "150": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "152": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "153": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "154": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "155": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "156": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "157": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "158": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "159": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "160": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "161": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "162": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "163": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "164": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "165": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "166": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "167": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "168": {
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "169": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "170": {
+      "content": "<caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "171": {
+      "content": "<thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "172": {
+      "content": "<tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "173": {
+      "content": "<tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "174": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "175": {
+      "content": "<th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "176": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "177": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "178": {
+      "content": "</caption>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "179": {
+      "content": "</thead>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "180": {
+      "content": "</tbody>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "181": {
+      "content": "</tfoot>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "182": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "183": {
+      "content": "</th>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "184": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "185": {
+      "content": "<h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "186": {
+      "content": "<h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "187": {
+      "content": "<h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "188": {
+      "content": "<h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "189": {
+      "content": "<h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "190": {
+      "content": "<h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "191": {
+      "content": "<blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "192": {
+      "content": "</h1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "193": {
+      "content": "</h2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "194": {
+      "content": "</h3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "195": {
+      "content": "</h4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "196": {
+      "content": "</h5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "197": {
+      "content": "</h6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "198": {
+      "content": "</blockquote>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "199": {
+      "content": "<strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "200": {
+      "content": "<em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "201": {
+      "content": "<b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "202": {
+      "content": "<i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "203": {
+      "content": "<u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "204": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "205": {
+      "content": "<sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "206": {
+      "content": "<sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "207": {
+      "content": "<code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "208": {
+      "content": "</strong>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "209": {
+      "content": "</em>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "210": {
+      "content": "</b>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "211": {
+      "content": "</i>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "212": {
+      "content": "</u>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "213": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "214": {
+      "content": "</sub>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "215": {
+      "content": "</sup>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "216": {
+      "content": "</code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255968": {
+      "content": "[toxicity=0]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255969": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255970": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255971": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255972": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255973": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255974": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255975": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255976": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255977": {
+      "content": "\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255978": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255979": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255980": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255981": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255982": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255983": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255984": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255985": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255986": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255987": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255988": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255989": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255990": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255991": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255992": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255993": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255994": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255995": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255996": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255997": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255998": {
+      "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255999": {
+      "content": "<unused99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "bos_token": "<bos>",
+  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "extra_special_tokens": {},
+  "model_max_length": 2048,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9969230769230769,
+    "total_flos": 82980012523520.0,
+    "train_loss": 1.2597844002178176,
+    "train_runtime": 2824.655,
+    "train_samples": 31200,
+    "train_samples_per_second": 11.046,
+    "train_steps_per_second": 0.086
+}

trainer_state.json

@@ -0,0 +1,385 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9969230769230769,
+  "eval_steps": 100,
+  "global_step": 243,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0041025641025641026,
+      "grad_norm": 210.52482598709943,
+      "learning_rate": 8e-08,
+      "loss": 18.5684,
+      "step": 1
+    },
+    {
+      "epoch": 0.020512820512820513,
+      "grad_norm": 186.8406143210285,
+      "learning_rate": 4e-07,
+      "loss": 17.1831,
+      "step": 5
+    },
+    {
+      "epoch": 0.041025641025641026,
+      "grad_norm": 49.91185411718416,
+      "learning_rate": 8e-07,
+      "loss": 9.5085,
+      "step": 10
+    },
+    {
+      "epoch": 0.06153846153846154,
+      "grad_norm": 9.332467552302559,
+      "learning_rate": 1.2e-06,
+      "loss": 2.0752,
+      "step": 15
+    },
+    {
+      "epoch": 0.08205128205128205,
+      "grad_norm": 0.8297806389960246,
+      "learning_rate": 1.6e-06,
+      "loss": 0.9159,
+      "step": 20
+    },
+    {
+      "epoch": 0.10256410256410256,
+      "grad_norm": 3.114341047106303,
+      "learning_rate": 2e-06,
+      "loss": 0.8841,
+      "step": 25
+    },
+    {
+      "epoch": 0.12307692307692308,
+      "grad_norm": 1.934423072293649,
+      "learning_rate": 1.9974051702905273e-06,
+      "loss": 0.8479,
+      "step": 30
+    },
+    {
+      "epoch": 0.14358974358974358,
+      "grad_norm": 1.6875673048587294,
+      "learning_rate": 1.9896341474445524e-06,
+      "loss": 0.8417,
+      "step": 35
+    },
+    {
+      "epoch": 0.1641025641025641,
+      "grad_norm": 1.7762270563382576,
+      "learning_rate": 1.976727260423982e-06,
+      "loss": 0.7891,
+      "step": 40
+    },
+    {
+      "epoch": 0.18461538461538463,
+      "grad_norm": 0.9578157871381868,
+      "learning_rate": 1.9587514915766123e-06,
+      "loss": 0.8344,
+      "step": 45
+    },
+    {
+      "epoch": 0.20512820512820512,
+      "grad_norm": 2.6943003842765423,
+      "learning_rate": 1.935800129020554e-06,
+      "loss": 0.7999,
+      "step": 50
+    },
+    {
+      "epoch": 0.22564102564102564,
+      "grad_norm": 0.8483257920517897,
+      "learning_rate": 1.907992282510675e-06,
+      "loss": 0.7871,
+      "step": 55
+    },
+    {
+      "epoch": 0.24615384615384617,
+      "grad_norm": 1.355665613118592,
+      "learning_rate": 1.8754722652995345e-06,
+      "loss": 0.7603,
+      "step": 60
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.3698034199623528,
+      "learning_rate": 1.8384088452007577e-06,
+      "loss": 0.7711,
+      "step": 65
+    },
+    {
+      "epoch": 0.28717948717948716,
+      "grad_norm": 0.9973813732040485,
+      "learning_rate": 1.7969943687415575e-06,
+      "loss": 0.7702,
+      "step": 70
+    },
+    {
+      "epoch": 0.3076923076923077,
+      "grad_norm": 2.067655192245599,
+      "learning_rate": 1.751443762949772e-06,
+      "loss": 0.7674,
+      "step": 75
+    },
+    {
+      "epoch": 0.3282051282051282,
+      "grad_norm": 0.7664884814735945,
+      "learning_rate": 1.7019934199557866e-06,
+      "loss": 0.7411,
+      "step": 80
+    },
+    {
+      "epoch": 0.3487179487179487,
+      "grad_norm": 1.0493906781125153,
+      "learning_rate": 1.6488999701978903e-06,
+      "loss": 0.7015,
+      "step": 85
+    },
+    {
+      "epoch": 0.36923076923076925,
+      "grad_norm": 0.9938125483952975,
+      "learning_rate": 1.5924389505977035e-06,
+      "loss": 0.7287,
+      "step": 90
+    },
+    {
+      "epoch": 0.38974358974358975,
+      "grad_norm": 1.0677317732124718,
+      "learning_rate": 1.5329033746173974e-06,
+      "loss": 0.7254,
+      "step": 95
+    },
+    {
+      "epoch": 0.41025641025641024,
+      "grad_norm": 0.8122971279562344,
+      "learning_rate": 1.4706022116196205e-06,
+      "loss": 0.7475,
+      "step": 100
+    },
+    {
+      "epoch": 0.4307692307692308,
+      "grad_norm": 0.7681446410916759,
+      "learning_rate": 1.4058587834217354e-06,
+      "loss": 0.7163,
+      "step": 105
+    },
+    {
+      "epoch": 0.4512820512820513,
+      "grad_norm": 0.766906823065794,
+      "learning_rate": 1.3390090863657047e-06,
+      "loss": 0.7117,
+      "step": 110
+    },
+    {
+      "epoch": 0.4717948717948718,
+      "grad_norm": 0.8487697397343301,
+      "learning_rate": 1.2704000476115078e-06,
+      "loss": 0.6475,
+      "step": 115
+    },
+    {
+      "epoch": 0.49230769230769234,
+      "grad_norm": 1.0050583245136004,
+      "learning_rate": 1.200387724703341e-06,
+      "loss": 0.7105,
+      "step": 120
+    },
+    {
+      "epoch": 0.5128205128205128,
+      "grad_norm": 0.7426930539258657,
+      "learning_rate": 1.1293354577522264e-06,
+      "loss": 0.7158,
+      "step": 125
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 0.8413604322965307,
+      "learning_rate": 1.0576119838245842e-06,
+      "loss": 0.6915,
+      "step": 130
+    },
+    {
+      "epoch": 0.5538461538461539,
+      "grad_norm": 0.5752843532661515,
+      "learning_rate": 9.85589523322443e-07,
+      "loss": 0.7209,
+      "step": 135
+    },
+    {
+      "epoch": 0.5743589743589743,
+      "grad_norm": 0.9645483285040541,
+      "learning_rate": 9.136418482863228e-07,
+      "loss": 0.6739,
+      "step": 140
+    },
+    {
+      "epoch": 0.5948717948717949,
+      "grad_norm": 0.7607550081366585,
+      "learning_rate": 8.42142342645646e-07,
+      "loss": 0.6717,
+      "step": 145
+    },
+    {
+      "epoch": 0.6153846153846154,
+      "grad_norm": 1.0589042706336858,
+      "learning_rate": 7.714620644833109e-07,
+      "loss": 0.6534,
+      "step": 150
+    },
+    {
+      "epoch": 0.6358974358974359,
+      "grad_norm": 0.7089491583646673,
+      "learning_rate": 7.019678203706163e-07,
+      "loss": 0.6696,
+      "step": 155
+    },
+    {
+      "epoch": 0.6564102564102564,
+      "grad_norm": 0.9341940371244601,
+      "learning_rate": 6.340202617660841e-07,
+      "loss": 0.6482,
+      "step": 160
+    },
+    {
+      "epoch": 0.676923076923077,
+      "grad_norm": 0.7411098123857883,
+      "learning_rate": 5.679720133572206e-07,
+      "loss": 0.6187,
+      "step": 165
+    },
+    {
+      "epoch": 0.6974358974358974,
+      "grad_norm": 0.8616495817851872,
+      "learning_rate": 5.041658430584852e-07,
+      "loss": 0.6386,
+      "step": 170
+    },
+    {
+      "epoch": 0.717948717948718,
+      "grad_norm": 0.7200165364613889,
+      "learning_rate": 4.429328831625565e-07,
+      "loss": 0.6567,
+      "step": 175
+    },
+    {
+      "epoch": 0.7384615384615385,
+      "grad_norm": 0.6168896507164103,
+      "learning_rate": 3.8459091187650726e-07,
+      "loss": 0.6644,
+      "step": 180
+    },
+    {
+      "epoch": 0.7589743589743589,
+      "grad_norm": 0.6719936032666863,
+      "learning_rate": 3.294427041611425e-07,
+      "loss": 0.6464,
+      "step": 185
+    },
+    {
+      "epoch": 0.7794871794871795,
+      "grad_norm": 0.8824680726642901,
+      "learning_rate": 2.777744604320705e-07,
+      "loss": 0.6362,
+      "step": 190
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.7384901847927622,
+      "learning_rate": 2.2985432127701941e-07,
+      "loss": 0.6385,
+      "step": 195
+    },
+    {
+      "epoch": 0.8205128205128205,
+      "grad_norm": 0.9346648012471711,
+      "learning_rate": 1.8593097589751316e-07,
+      "loss": 0.6423,
+      "step": 200
+    },
+    {
+      "epoch": 0.841025641025641,
+      "grad_norm": 0.7741404463056779,
+      "learning_rate": 1.4623237149661137e-07,
+      "loss": 0.6425,
+      "step": 205
+    },
+    {
+      "epoch": 0.8615384615384616,
+      "grad_norm": 0.7837940282138485,
+      "learning_rate": 1.1096453031056264e-07,
+      "loss": 0.5925,
+      "step": 210
+    },
+    {
+      "epoch": 0.882051282051282,
+      "grad_norm": 0.6434405301070071,
+      "learning_rate": 8.031048042356392e-08,
+      "loss": 0.6155,
+      "step": 215
+    },
+    {
+      "epoch": 0.9025641025641026,
+      "grad_norm": 0.6950331917340359,
+      "learning_rate": 5.442930591433992e-08,
+      "loss": 0.6336,
+      "step": 220
+    },
+    {
+      "epoch": 0.9230769230769231,
+      "grad_norm": 0.7624734080498339,
+      "learning_rate": 3.345532126395578e-08,
+      "loss": 0.628,
+      "step": 225
+    },
+    {
+      "epoch": 0.9435897435897436,
+      "grad_norm": 0.7335512264577648,
+      "learning_rate": 1.7497374309405344e-08,
+      "loss": 0.6125,
+      "step": 230
+    },
+    {
+      "epoch": 0.9641025641025641,
+      "grad_norm": 0.7687162657729733,
+      "learning_rate": 6.6382813604083375e-09,
+      "loss": 0.6287,
+      "step": 235
+    },
+    {
+      "epoch": 0.9846153846153847,
+      "grad_norm": 0.7525426453572169,
+      "learning_rate": 9.343974109685682e-10,
+      "loss": 0.6188,
+      "step": 240
+    },
+    {
+      "epoch": 0.9969230769230769,
+      "step": 243,
+      "total_flos": 82980012523520.0,
+      "train_loss": 1.2597844002178176,
+      "train_runtime": 2824.655,
+      "train_samples_per_second": 11.046,
+      "train_steps_per_second": 0.086
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 243,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 82980012523520.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f4de4053559e16134d5f89446d7335a2798a27dc0544bbc0b5c624547b07ebe
+size 7160