experiments/results/norman_nom_improved.json

{
  "dataset": "norman",
  "split": "perturbation",
  "reward": "cosine",
  "n_candidates": 105,
  "n_targets": 22,
  "random_top5": 0.047619047619047616,
  "methods": {
    "PIVOT-ranking": {
      "top1": 0.18181818181818182,
      "top5": 0.22727272727272727,
      "ndcg": 0.237863170222463,
      "func_top5": 0.8636363636363636,
      "func_ndcg": 0.6379857109126424,
      "top5_ci": [
        0.22727272727272727,
        0.045454545454545456,
        0.4090909090909091
      ],
      "_top5_per": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        1.0,
        0.0,
        1.0,
        0.0,
        1.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "med_rank": 26.0
    },
    "PIVOT-guidance(random-init)": {
      "top1": 0.13636363636363635,
      "top5": 0.22727272727272727,
      "ndcg": 0.20036677015736315,
      "func_top5": 0.8181818181818182,
      "func_ndcg": 0.5717396066885763,
      "top5_ci": [
        0.22727272727272727,
        0.045454545454545456,
        0.4090909090909091
      ],
      "_top5_per": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        1.0,
        0.0,
        1.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0
      ],
      "med_rank": 27.0
    },
    "EndpointMLP+ranking": {
      "top1": 0.045454545454545456,
      "top5": 0.13636363636363635,
      "ndcg": 0.13489139848118523,
      "func_top5": 0.9545454545454546,
      "func_ndcg": 0.5888636133223321,
      "top5_ci": [
        0.13636363636363635,
        0.0,
        0.2727272727272727
      ],
      "_top5_per": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "med_rank": 74.5
    },
    "LinearResponse+ranking": {
      "top1": 0.045454545454545456,
      "top5": 0.13636363636363635,
      "ndcg": 0.1028117957791206,
      "func_top5": 0.8181818181818182,
      "func_ndcg": 0.5703934469254858,
      "top5_ci": [
        0.13636363636363635,
        0.0,
        0.2727272727272727
      ],
      "_top5_per": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "med_rank": 53.0
    },
    "Additive+ranking": {
      "top1": 0.0,
      "top5": 0.0,
      "ndcg": 0.0,
      "func_top5": 0.9545454545454546,
      "func_ndcg": 0.5342699870208798,
      "top5_ci": [
        0.0,
        0.0,
        0.0
      ],
      "_top5_per": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "med_rank": 40.5
    },
    "NearestPerturbationCentroid+ranking": {
      "top1": 0.18181818181818182,
      "top5": 0.18181818181818182,
      "ndcg": 0.181818181818,
      "func_top5": 0.9090909090909091,
      "func_ndcg": 0.5876837637320901,
      "top5_ci": [
        0.18181818181818182,
        0.045454545454545456,
        0.36363636363636365
      ],
      "_top5_per": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        1.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "med_rank": 35.5
    },
    "Random+ranking": {
      "top1": 0.0,
      "top5": 0.045454545454545456,
      "ndcg": 0.030723528797806977,
      "func_top5": 0.7727272727272727,
      "func_ndcg": 0.24888537092571922,
      "top5_ci": [
        0.045454545454545456,
        0.0,
        0.13636363636363635
      ],
      "_top5_per": [
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        1.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "med_rank": 57.5
    }
  },
  "significance": {
    "PIVOT-ranking_vs_PIVOT-guidance(random-init)": {
      "p": 1.0,
      "mean_diff_top5": 0.0
    },
    "PIVOT-ranking_vs_EndpointMLP+ranking": {
      "p": 0.3268365817091454,
      "mean_diff_top5": 0.09090909090909091
    },
    "PIVOT-ranking_vs_LinearResponse+ranking": {
      "p": 0.12643678160919541,
      "mean_diff_top5": 0.09090909090909091
    },
    "PIVOT-ranking_vs_Additive+ranking": {
      "p": 0.010994502748625687,
      "mean_diff_top5": 0.22727272727272727
    },
    "PIVOT-ranking_vs_NearestPerturbationCentroid+ranking": {
      "p": 0.25287356321839083,
      "mean_diff_top5": 0.045454545454545456
    },
    "PIVOT-ranking_vs_Random+ranking": {
      "p": 0.03298350824587706,
      "mean_diff_top5": 0.18181818181818182
    }
  }
}