data/data_context.json

{
  "completion": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 1.0
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.9714285714
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.945
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.9378571429
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.9357142857
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.9328571429
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.9214285714
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.9007142857
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.8907142857
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.8828571429
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.8728571429
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.8578571429
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.845
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.7964285714
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.7935714286
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.7192857143
    }
  ],
  "compilation_class_wise": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.7942857143
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.7414285714
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.7385714286
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.7314285714
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.7171428571
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.6978571429
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 0.6607142857
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.6592857143
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.6414285714
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.58
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.4814285714
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.45
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.39
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.3692857143
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.3457142857
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.215
    }
  ],
  "compilation_test_wise": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 0.5035714286
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.4202826585
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.3443277311
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.3405987395
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.3387079832
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.3183823529
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.3121848739
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.2858193277
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.2283088235
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.2240546218
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.1466911765
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.1128676471
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.03125
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.0147058824
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.0125
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.0
    }
  ],
  "pass_class_wise": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.7832360347
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.723699056
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.715291943
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.7033228696
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.6855203826
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.6808480861
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 0.6545897285
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.6417690022
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.6293667264
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.5674101922
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.4741970721
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.4489821662
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.3864123669
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.3599216366
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.3272885064
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.2139054163
    }
  ],
  "pass_test_wise": [
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-4o-2024-05-13",
      "Pass_at_1": 0.3438179726
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.3047552867
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.3032497787
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.2941156144
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.2544265255
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.2393000344
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.2028454735
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.1966660863
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.1877858469
    },
    {
      "Context": "selective",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.1669267449
    },
    {
      "Context": "maximum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.1370849195
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-33b-instruct",
      "Pass_at_1": 0.1123372221
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "gpt-3.5-turbo-1106",
      "Pass_at_1": 0.03125
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "WizardCoder-15B-V1.0",
      "Pass_at_1": 0.0147058824
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "deepseek-coder-6.7b-instruct",
      "Pass_at_1": 0.0125
    },
    {
      "Context": "minimum",
      "Method": "holistic",
      "Model": "Phind-CodeLlama-34B-v2",
      "Pass_at_1": 0.0
    }
  ]
}