Route2/route2_results.json

{
  "retrieval": {
    "SIGMA_B": {
      "mrr10": 0.92306,
      "r1": 0.8856,
      "r5": 0.96956,
      "r10": 0.97886,
      "median_rank": 1.0,
      "pool_size": 22046
    },
    "CodeBERT_FT": {
      "mrr10": 0.96622,
      "r1": 0.94484,
      "r5": 0.99129,
      "r10": 0.99356,
      "median_rank": 1.0,
      "pool_size": 22046
    },
    "delta_mrr": -0.04316
  },
  "generation": {
    "sigma_gen_hist": {
      "tr_loss": [
        4.332911533272467,
        3.513244101578118,
        3.232151625534834,
        3.0188788402276594,
        2.876566329031201,
        2.774567126666569,
        2.7000688419086245,
        2.6459983390151094,
        2.61164868748713,
        2.5937092144469105
      ],
      "va_loss": [
        4.333631096980431,
        4.033719047488412,
        3.887238296438657,
        3.721125888444704,
        3.652845814161021,
        3.598194845310101,
        3.560626884405384,
        3.542548859641584,
        3.5358788442590083,
        3.5271108577827133
      ],
      "va_ppl": [
        76.2205489029526,
        56.470537811754056,
        48.77599550446955,
        41.31087937681806,
        38.58431374299519,
        36.53222856246678,
        35.18524731577227,
        34.5548825857037,
        34.32516793907655,
        34.02552090105466
      ]
    },
    "cbert_gen_hist": {
      "tr_loss": [
        4.357259073945283,
        3.5188005480055473,
        3.1955701593955923,
        2.988982406454519,
        2.858495755219526,
        2.7642144319277264,
        2.69454125002109,
        2.644378586365032,
        2.6121866509032623,
        2.5950699153240633
      ],
      "va_loss": [
        4.360623060350565,
        4.023048199407739,
        3.830441037275766,
        3.7134811096919123,
        3.6445059282146337,
        3.6011066965343077,
        3.5686047597707726,
        3.551744504028521,
        3.532704999526294,
        3.5293681702964337
      ],
      "va_ppl": [
        78.30590853372148,
        55.871152941678396,
        46.08285801115232,
        40.996270932086794,
        38.263863085402065,
        36.63876000351066,
        35.46707352597844,
        34.874102464608406,
        34.216397886548734,
        34.10241388827117
      ]
    },
    "gen_samples": [
      {
        "prompt": "Calculate the fibonacci number",
        "sigma": "def get_best(of, x):\n    \"\"\"Calculate the year of distribution.\"\"\"\n    if len(x, t.shape):\n     q = np.shape(cos_ind)\n    for t in range(x, n):\n        if s <= 0:\n ",
        "codebert": "def rot(self, X):\n        \"\"\"Return the increment and posterior of all details.\"\"\"\n        return self.bits(np(\"%s\")\n        return self.read_index_value(self._num)\n        else:\n   "
      },
      {
        "prompt": "Sort a list using quicksort algorithm",
        "sigma": "def _sqr_valid(self, p):\n    \"\"\"Build the input a region of shape,\n    a matrix in the grid(d = float)\n    # Find the lines to make a point.\n    # and '''\n\n\r\n    if isinstance(value, x1):\r\n   ",
        "codebert": "def _rc_words(self, node):\n    \"\"\"Returns the file when a dict.\"\"\"\n    if isinstance(log(\" not necessary\") == 'f'):\n     # self._match()\n  if _swith(\"The tag\" in self.format(self._case):\n   # Popon: d"
      },
      {
        "prompt": "Read a file and return its contents as string",
        "sigma": "def _format_to(path):\n    \"\"\"Helper a file from the doi into `file`.\n   Returns:\n   \"\"\"\n\n    # Note:\n    return raw_file['test__', 'f'\n    # Use ``object'\n    return (arg.strip)\n   # Runtime =",
        "codebert": "def _parse_file(self, filename):\n        \"\"\"\n        Render a single file object\n        \"\"\"\n        try:\n            for path in self.path['wargs']\n     "
      },
      {
        "prompt": "Convert a dictionary to a JSON string",
        "sigma": "def _code(self, placeholder):\n idd\"\"\"' if a string and return None. \"\"\"\n    if not None:\n    return '{_list':\n    return [int(node.join('OTO')\n   info = 'name['join(node', str)\n    data = '%s}\n ",
        "codebert": "def _read_names(self, type):\n    \"\"\"Convert a string with single style object.\n\n    Returns the value of `line` object and if not be a\n    \"\"\"\n\n    if self.values:\n     if not None:\n      # type in se"
      },
      {
        "prompt": "Find the maximum element in a list",
        "sigma": "def _info_field(self, value):\n    '''Returns a list for this is the packet\"\"\"\n    if not None:\n    return self._open(\n        return raw_value)\n    return not None:\n    # we coerree of the child\n   ",
        "codebert": "def get_field(self, node):\n    '''Return the raw tokens.\"\"\"\n    for key in value in node.items:\n    else = self._line_obj(index)\n    else: ast.value(self._text)\n    elif self.isfocol == 1:\n   "
      },
      {
        "prompt": "Calculate the factorial of a number",
        "sigma": "def calculate_p(self, width=None):\n    \"\"\"Calculate the water of a normalized time and\n    '''\n\n    n = np.term['sub1\n    return np.scplot_vON(x, x)\n    return self.fasttype(float)\n    return self",
        "codebert": "def compute(self):\n        \"\"\"\n        Updates the time-block over all one of a column\n        \"\"\"\n        s = self.build(self._coist, self.left_count)\n        if self.start"
      }
    ]
  }
}