data/data_method.json

{
  "completion": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 1.0
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.9378571429
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.9357142857
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.9214285714
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.8642857143
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.8535714286
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.8471428571
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.845
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.8392857143
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.8192857143
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.7964285714
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.7485714286
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.6378571429
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.5992857143
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.5971428571
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.5057142857
    }
  ],
  "compilation_class_wise": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.7942857143
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.7414285714
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.725
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.7171428571
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.7
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.6978571429
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.6828571429
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.6814285714
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.6764285714
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 0.6607142857
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.58
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.5778571429
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.5021428571
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.4985714286
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.4335714286
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.3507142857
    }
  ],
  "compilation_test_wise": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 0.5035714286
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.4202826585
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.3443277311
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.3378676471
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.3183823529
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.3121848739
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.2836134454
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.2621848739
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.2240546218
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.1930147059
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.0955357143
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.0932773109
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.0701680672
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.0352941176
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.0
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.0
    }
  ],
  "pass_class_wise": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.7832360347
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.723699056
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.7036481325
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.6855203826
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.6846699639
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.6808480861
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.6772858762
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.6547244155
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.6547232007
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 0.6545897285
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.5674101922
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.5603969625
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.4878321662
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.4863639752
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.4261740357
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.3468474087
    }
  ],
  "pass_test_wise": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 0.3438179726
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.3047552867
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.2941156144
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.2544265255
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.2224382166
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.2083516025
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.2028454735
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.1967821219
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.1930147059
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.1669267449
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.0714792814
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.061012122
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.0514928193
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.0350620781
    },
    {
      "Context": "selective",
      "Method": "independent",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.0
    },
    {
      "Context": "selective",
      "Method": "incremental",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.0
    }
  ]
}