toy-cmdp-v2/results.json

{
  "study_type": "independent synthetic boundary test; not BiGym/LIBERO/robot evaluation",
  "design": {
    "seeds": 4,
    "offline_train_episodes_per_seed": 180,
    "evaluation_tasks": 3,
    "evaluation_episodes_per_task_seed": 60,
    "scheduler_rates": [
      1,
      2,
      3,
      4
    ]
  },
  "base_policy_hash_before": "1207a19ab7f917ec0b25482df644bfd90b364a4dc57e2d88f6b0cc7b406f918d",
  "base_policy_hash_after": "1207a19ab7f917ec0b25482df644bfd90b364a4dc57e2d88f6b0cc7b406f918d",
  "base_policy_unchanged": true,
  "aggregates": {
    "base": {
      "success_rate": 1.0,
      "mean_steps": 58.01111111111111,
      "mean_speedup_vs_base": 1.0,
      "mean_selected_rate": 1.0
    },
    "fixed4": {
      "success_rate": 0.0,
      "mean_steps": 41.71666666666667,
      "mean_speedup_vs_base": 1.3708886205045625,
      "mean_selected_rate": 4.0
    },
    "recurrent": {
      "success_rate": 0.9777777777777777,
      "mean_steps": 47.045833333333334,
      "mean_speedup_vs_base": 1.255138588496929,
      "mean_selected_rate": 2.4971000392354563
    }
  },
  "world_models": {
    "recurrent_parameters": 2041.0,
    "mlp_parameters": 97.0,
    "recurrent_train_mse": 0.00026125604563276283,
    "mlp_train_mse": 0.0004586470386129804,
    "recurrent_test_mse": 0.00027383754786569625,
    "mlp_test_mse": 0.0004723040256067179
  },
  "deviation_diagnostic": {
    "recurrent_spearman_predicted_vs_true_deviation": 0.7405268902881142,
    "mlp_spearman_predicted_vs_true_deviation": -0.010582534716543152,
    "recurrent_binned_deviation_vs_success": -0.8363636363636363,
    "mlp_binned_deviation_vs_success": 0.22424242424242422,
    "safe_fraction": 0.4975,
    "true_deviation_median": 0.14737405085187963
  },
  "per_seed_world_models": [
    {
      "seed": 0,
      "recurrent_parameters": 2041,
      "mlp_parameters": 97,
      "recurrent_train_mse": 0.0001863662910182029,
      "mlp_train_mse": 0.00028698742971755564,
      "recurrent_test_mse": 0.00018156242731492966,
      "mlp_test_mse": 0.00028945444501005113
    },
    {
      "seed": 1,
      "recurrent_parameters": 2041,
      "mlp_parameters": 97,
      "recurrent_train_mse": 0.00019247485033702105,
      "mlp_train_mse": 0.0004385770298540592,
      "recurrent_test_mse": 0.00019988064013887197,
      "mlp_test_mse": 0.0004780442104674876
    },
    {
      "seed": 2,
      "recurrent_parameters": 2041,
      "mlp_parameters": 97,
      "recurrent_train_mse": 0.0003118192544206977,
      "mlp_train_mse": 0.0005170261720195413,
      "recurrent_test_mse": 0.00027093858807347715,
      "mlp_test_mse": 0.0004079154459759593
    },
    {
      "seed": 3,
      "recurrent_parameters": 2041,
      "mlp_parameters": 97,
      "recurrent_train_mse": 0.0003543637867551297,
      "mlp_train_mse": 0.0005919975228607655,
      "recurrent_test_mse": 0.0004429685359355062,
      "mlp_test_mse": 0.0007138020009733737
    }
  ],
  "per_seed_diagnostics": [
    {
      "seed": 0,
      "recurrent_spearman_predicted_vs_true_deviation": 0.7914832362631643,
      "mlp_spearman_predicted_vs_true_deviation": -0.15626942678891764,
      "recurrent_binned_deviation_vs_success": -0.8545454545454544,
      "mlp_binned_deviation_vs_success": 0.32121212121212117,
      "safe_fraction": 0.49,
      "true_deviation_median": 0.14761173104586012
    },
    {
      "seed": 1,
      "recurrent_spearman_predicted_vs_true_deviation": 0.8276342021689257,
      "mlp_spearman_predicted_vs_true_deviation": -0.022739871480787273,
      "recurrent_binned_deviation_vs_success": -0.8545454545454544,
      "mlp_binned_deviation_vs_success": 0.10303030303030303,
      "safe_fraction": 0.48857142857142855,
      "true_deviation_median": 0.14826513363357416
    },
    {
      "seed": 2,
      "recurrent_spearman_predicted_vs_true_deviation": 0.74997830024726,
      "mlp_spearman_predicted_vs_true_deviation": 0.021084672767553752,
      "recurrent_binned_deviation_vs_success": -0.8181818181818182,
      "mlp_binned_deviation_vs_success": 0.33333333333333326,
      "safe_fraction": 0.48142857142857143,
      "true_deviation_median": 0.15015068633506076
    },
    {
      "seed": 3,
      "recurrent_spearman_predicted_vs_true_deviation": 0.593011822473107,
      "mlp_spearman_predicted_vs_true_deviation": 0.11559448663597856,
      "recurrent_binned_deviation_vs_success": -0.8181818181818182,
      "mlp_binned_deviation_vs_success": 0.13939393939393938,
      "safe_fraction": 0.53,
      "true_deviation_median": 0.14346865239302342
    }
  ]
}