fim-competitive-v1/report.json

{
  "bins": [
    "8_token_ish",
    "16_token_ish",
    "32_token_ish",
    "64_token_ish"
  ],
  "contract_id": "wisp-short-hole-fim-competitive-v1",
  "descriptive_intervals": {
    "Qwen__Qwen2.5-Coder-0.5B": {
      "16_token_ish": [
        0.038461538461538464,
        0.08875739644970414
      ],
      "32_token_ish": [
        0.0,
        0.02416918429003021
      ],
      "64_token_ish": [
        0.0,
        0.012468827930174564
      ],
      "8_token_ish": [
        0.1092896174863388,
        0.19680851063829788
      ]
    },
    "bigcode__tiny_starcoder_py": {
      "16_token_ish": [
        0.0,
        0.0125
      ],
      "32_token_ish": [
        0.0,
        0.010050251256281407
      ],
      "64_token_ish": [
        0.0,
        0.0
      ],
      "8_token_ish": [
        0.007614213197969543,
        0.04895104895104895
      ]
    },
    "codeparrot__codeparrot-small": {
      "16_token_ish": [
        0.0027548209366391185,
        0.022950819672131147
      ],
      "32_token_ish": [
        0.0,
        0.010050251256281407
      ],
      "64_token_ish": [
        0.0,
        0.0
      ],
      "8_token_ish": [
        0.006369426751592357,
        0.03990610328638498
      ]
    },
    "ibm-granite__granite-4.0-350m-base": {
      "16_token_ish": [
        0.0,
        0.024305555555555556
      ],
      "32_token_ish": [
        0.0,
        0.01557632398753894
      ],
      "64_token_ish": [
        0.0,
        0.0
      ],
      "8_token_ish": [
        0.01483679525222552,
        0.050793650793650794
      ]
    },
    "wisp": {
      "16_token_ish": [
        0.0025974025974025974,
        0.02464788732394366
      ],
      "32_token_ish": [
        0.0,
        0.0
      ],
      "64_token_ish": [
        0.0,
        0.0
      ],
      "8_token_ish": [
        0.00974025974025974,
        0.04814814814814815
      ]
    }
  },
  "method": "paired nonparametric repository cluster bootstrap",
  "model_replicate_macro_means": {
    "Qwen__Qwen2.5-Coder-0.5B": {
      "descriptive95": [
        0.044585987261146494,
        0.06975308641975309
      ],
      "mean": 0.05684153680081007
    },
    "bigcode__tiny_starcoder_py": {
      "descriptive95": [
        0.0037128712871287127,
        0.01419558359621451
      ],
      "mean": 0.008733842367255456
    },
    "codeparrot__codeparrot-small": {
      "descriptive95": [
        0.003633720930232558,
        0.015277777777777777
      ],
      "mean": 0.009366252755936293
    },
    "ibm-granite__granite-4.0-350m-base": {
      "descriptive95": [
        0.005952380952380952,
        0.018493150684931507
      ],
      "mean": 0.01170650003827442
    },
    "wisp": {
      "descriptive95": [
        0.005028735632183908,
        0.01551094890510949
      ],
      "mean": 0.010161879398330735
    }
  },
  "models": [
    "Qwen__Qwen2.5-Coder-0.5B",
    "bigcode__tiny_starcoder_py",
    "codeparrot__codeparrot-small",
    "ibm-granite__granite-4.0-350m-base",
    "wisp"
  ],
  "n_repositories": 24,
  "paired_deltas": {
    "Qwen__Qwen2.5-Coder-0.5B": {
      "bonferroni_onesided_95_lower": -0.06115107913669065,
      "delta_mean": -0.04667965740247933,
      "descriptive95": [
        -0.06,
        -0.0339943342776204
      ],
      "lower_exceeds_zero": false
    },
    "bigcode__tiny_starcoder_py": {
      "bonferroni_onesided_95_lower": -0.005033557046979867,
      "delta_mean": 0.0014280370310752804,
      "descriptive95": [
        -0.004629629629629629,
        0.007472826086956521
      ],
      "lower_exceeds_zero": false
    },
    "codeparrot__codeparrot-small": {
      "bonferroni_onesided_95_lower": -0.0024590163934426236,
      "delta_mean": 0.0007956266423944429,
      "descriptive95": [
        -0.002252252252252252,
        0.004325259515570935
      ],
      "lower_exceeds_zero": false
    },
    "ibm-granite__granite-4.0-350m-base": {
      "bonferroni_onesided_95_lower": -0.008384146341463417,
      "delta_mean": -0.0015446206399436838,
      "descriptive95": [
        -0.007812499999999998,
        0.004451038575667655
      ],
      "lower_exceeds_zero": false
    }
  },
  "repositories": [
    "0xPlaygrounds/rig",
    "D4Vinci/Scrapling",
    "DataWithBaraa/sql-data-warehouse-project",
    "DietrichGebert/ponytail",
    "EpicGames/raddebugger",
    "Fission-AI/OpenSpec",
    "JuliusBrussee/caveman",
    "Netflix/maestro",
    "OpenCut-app/OpenCut",
    "TecharoHQ/anubis",
    "asg017/sqlite-vec",
    "browser-use/browser-use",
    "cloudwego/eino",
    "dockur/windows",
    "henrygd/beszel",
    "j178/prek",
    "jackwener/OpenCLI",
    "microsoft/RustTraining",
    "microsoft/markitdown",
    "modelcontextprotocol/java-sdk",
    "nicbarker/clay",
    "steipete/agent-scripts",
    "unitycatalog/unitycatalog",
    "upstash/context7"
  ],
  "samples": 10000,
  "schema_version": 1,
  "seed": 20260731
}