summary.json

{
  "config": {
    "local_samples_dir": "data_samples/redmod_hf_streaming",
    "merge_web": false,
    "classifier": "distilbert",
    "seed": 0,
    "val_fraction": 0.2,
    "target_model": "./qwen3p5base",
    "generator": "hf",
    "num_prompts": 3000,
    "max_new_tokens": 512,
    "gen_temperature": 0.8,
    "top_p": 0.9,
    "gen_seed": 0,
    "hf_revision": null,
    "hf_model_name": "distilbert/distilbert-base-uncased",
    "hf_epochs": 3,
    "hf_batch_size": 64,
    "hf_lr": 2e-05,
    "hf_weight_decay": 0.01,
    "hf_max_length": 256,
    "hf_pretrained_dir": null,
    "bootstrap": false,
    "n_boot": 300,
    "prompts_style": "unconditional"
  },
  "categories": [
    "cci4_extras_stem",
    "dclm_pro",
    "diverse_qa_filtered",
    "extras",
    "megamathweb_pro",
    "nemotron_synth_code",
    "retrieve_pile",
    "sft_code_proc",
    "sft_math_proc",
    "textbooks_expanded"
  ],
  "val_metrics": {
    "val_acc": 0.9222,
    "temperature": 1.1051709180756477,
    "model_name": "distilbert/distilbert-base-uncased"
  },
  "confusion_matrix": [
    [
      0.977,
      0.007,
      0.0,
      0.006,
      0.0,
      0.0,
      0.01,
      0.0,
      0.0,
      0.0
    ],
    [
      0.003,
      0.801,
      0.094,
      0.01,
      0.001,
      0.0,
      0.091,
      0.0,
      0.0,
      0.0
    ],
    [
      0.0,
      0.111,
      0.783,
      0.011,
      0.01,
      0.0,
      0.085,
      0.0,
      0.0,
      0.0
    ],
    [
      0.013,
      0.015,
      0.01,
      0.93,
      0.013,
      0.001,
      0.018,
      0.0,
      0.0,
      0.0
    ],
    [
      0.001,
      0.003,
      0.0,
      0.009,
      0.978,
      0.0,
      0.007,
      0.0,
      0.0,
      0.002
    ],
    [
      0.0,
      0.0,
      0.0,
      0.0,
      0.001,
      0.999,
      0.0,
      0.0,
      0.0,
      0.0
    ],
    [
      0.007,
      0.125,
      0.088,
      0.009,
      0.008,
      0.0,
      0.761,
      0.0,
      0.0,
      0.002
    ],
    [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      1.0,
      0.0,
      0.0
    ],
    [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      1.0,
      0.0
    ],
    [
      0.0,
      0.002,
      0.0,
      0.0,
      0.002,
      0.0,
      0.003,
      0.0,
      0.0,
      0.993
    ]
  ],
  "pbar": [
    0.21359796822071075,
    0.04607619345188141,
    0.0568278431892395,
    0.43409791588783264,
    0.1079854890704155,
    0.004097343422472477,
    0.125635027885437,
    0.0022735290694981813,
    0.002514379099011421,
    0.0068938578478991985
  ],
  "priors": {
    "point": [
      0.21129262330333382,
      0.01759996867924386,
      0.04850116596714966,
      0.46227047717098,
      0.10257029380532197,
      0.0036387569878495794,
      0.14289059730481754,
      0.0022735743550584405,
      0.002514424384571679,
      0.0064481180416733775
    ],
    "mean": [
      0.21129262330333382,
      0.01759996867924386,
      0.04850116596714966,
      0.46227047717098,
      0.10257029380532197,
      0.0036387569878495794,
      0.14289059730481754,
      0.0022735743550584405,
      0.002514424384571679,
      0.0064481180416733775
    ],
    "ci_lo": [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0
    ],
    "ci_hi": [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0
    ]
  },
  "unknown": {
    "mode": "threshold",
    "metric": "maxprob",
    "threshold": 0.9,
    "mean_probability": 0.13603055477142334,
    "ci_lo": 0.13603055477142334,
    "ci_hi": 0.13603055477142334
  },
  "categories_with_unknown": [
    "cci4_extras_stem",
    "dclm_pro",
    "diverse_qa_filtered",
    "extras",
    "megamathweb_pro",
    "nemotron_synth_code",
    "retrieve_pile",
    "sft_code_proc",
    "sft_math_proc",
    "textbooks_expanded",
    "Unknown"
  ],
  "priors_with_unknown": {
    "point": [
      0.18255037053627193,
      0.015205835175846642,
      0.041903525453577416,
      0.39938756770696104,
      0.08861759983591613,
      0.0031437748561140076,
      0.12345311008182316,
      0.00196429877422576,
      0.002172385840607599,
      0.005570976967232924,
      0.13603055477142334
    ],
    "mean": [
      0.18255037053627193,
      0.015205835175846642,
      0.041903525453577416,
      0.39938756770696104,
      0.08861759983591613,
      0.0031437748561140076,
      0.12345311008182316,
      0.00196429877422576,
      0.002172385840607599,
      0.005570976967232924,
      0.13603055477142334
    ],
    "ci_lo": [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.13603055477142334
    ],
    "ci_hi": [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.13603055477142334
    ]
  },
  "pbar_with_unknown": [
    0.18454211950302124,
    0.039808422327041626,
    0.04909751936793327,
    0.37504732608795166,
    0.09329616278409958,
    0.0035399794578552246,
    0.10854482650756836,
    0.001964259659871459,
    0.002172346692532301,
    0.005956082604825497,
    0.13603055477142334
  ]
}