naver-psy-classifier-gemma4-12b

LoRA adapter and per-field linear heads on the Gemma 4 12B text decoder for the nsb-v2 taxonomy of Naver 종목토론실 posts (naver-psy, docs/classifier-plan.md).

• Agreement figures are agreement with the teacher labels, not accuracy against ground truth.
• Input: nsb-input-v1 render, segments joined with newlines, suffix ### LABELS, one BOS.
• Output: raw logits per field; calibrated with temperatures.json, then gated (training/taxonomy.py).
• No training data, predictions or post text are in this repository.

Setting	Value
arm	B-last
checkpoint	base
pool	last
epochs	2.0
max_len	2048
lora_r	32
lora_alpha	32
lr_lora	0.0002
lr_heads	0.001
seed	20261017
data_version	v20261019-1009
data_manifest_sha256	b3bf46a948289240f59e08ca95c654022a6f748636d483540e8c4d46b35ed8c7

Metrics

metrics

{
 "version": "nsb-eval-v1",
 "metrics_version": "nsb-metrics-v1",
 "taxonomy_version": "nsb-v2",
 "macro_f1_exclude": {
  "emotion_fine": [
   "mixed"
  ]
 },
 "temperatures": {
  "relevance": 1.0902,
  "target": 1.1885,
  "direction": 1.2589,
  "sarcasm": 1.0902,
  "behavior": 1.2232,
  "pnl": 1.1548,
  "emotion_fine": 1.2589,
  "crowd": 1.1885,
  "commercial": 1.1885,
  "emotion_valence": 1.3335
 },
 "splits": {
  "val": {
   "n": 3014,
   "n_eval": 2237,
   "variants": {
    "raw_uncalibrated": {
     "relevance": {
      "n": 3014,
      "masked": 0,
      "accuracy": 0.9797611147976112,
      "macro_f1": 0.808889194738538,
      "kappa": 0.7876187362752248,
      "log_loss": 0.05375395597967696,
      "ece": 0.007202623891907931,
      "per_class": {
       "market": {
        "support": 2857,
        "predicted": 2874,
        "tp": 2840,
        "precision": 0.988169798190675,
        "recall": 0.9940497024851243,
        "f1": 0.9911010294887455
       },
       "politics": {
        "support": 56,
        "predicted": 53,
        "tp": 48,
        "precision": 0.9056603773584906,
        "recall": 0.8571428571428571,
        "f1": 0.8807339449541285
       },
       "off_topic": {
        "support": 38,
        "predicted": 38,
        "tp": 24,
        "precision": 0.631578947368421,
        "recall": 0.631578947368421,
        "f1": 0.631578947368421
       },
       "unclear": {
        "support": 63,
        "predicted": 49,
        "tp": 41,
        "precision": 0.8367346938775511,
        "recall": 0.6507936507936508,
        "f1": 0.7321428571428571
       }
      }
     },
     "target": {
      "n": 2857,
      "masked": 157,
      "accuracy": 0.943997199859993,
      "macro_f1": 0.7996162153170646,
      "kappa": 0.784795484622095,
      "log_loss": 0.16226858313157314,
      "ece": 0.017461389347822973,
      "per_class": {
       "this_stock": {
        "support": 2435,
        "predicted": 2459,
        "tp": 2380,
        "precision": 0.9678731191541277,
        "recall": 0.9774127310061602,
        "f1": 0.9726195341234164
       },
       "other_stock": {
        "support": 113,
        "predicted": 92,
        "tp": 68,
        "precision": 0.7391304347826086,
        "recall": 0.6017699115044248,
        "f1": 0.6634146341463415
       },
       "sector_theme": {
        "support": 101,
        "predicted": 109,
        "tp": 81,
        "precision": 0.7431192660550459,
        "recall": 0.801980198019802,
        "f1": 0.7714285714285715
       },
       "market_overall": {
        "support": 172,
        "predicted": 163,
        "tp": 142,
        "precision": 0.8711656441717791,
        "recall": 0.8255813953488372,
        "f1": 0.8477611940298507
       },
       "none": {
        "support": 36,
        "predicted": 34,
        "tp": 26,
        "precision": 0.7647058823529411,
        "recall": 0.7222222222222222,
        "f1": 0.7428571428571429
       }
      }
     },
     "direction": {
      "n": 2821,
      "masked": 193,
      "accuracy": 0.8883374689826302,
      "macro_f1": 0.7937674531232581,
      "kappa": 0.8266856998417065,
      "log_loss": 0.3094536615419838,
      "ece": 0.03246592389650786,
      "per_class": {
       "bullish": {
        "support": 907,
        "predicted": 896,
        "tp": 812,
        "precision": 0.90625,
        "recall": 0.8952590959206174,
        "f1": 0.9007210205213533
       },
       "bearish": {
        "support": 560,
        "predicted": 565,
        "tp": 483,
        "precision": 0.8548672566371681,
        "recall": 0.8625,
        "f1": 0.8586666666666667
       },
       "neutral_range": {
        "support": 32,
        "predicted": 27,
        "tp": 21,
        "precision": 0.7777777777777778,
        "recall": 0.65625,
        "f1": 0.711864406779661
       },
       "mixed": {
        "support": 25,
        "predicted": 22,
        "tp": 14,
        "precision": 0.6363636363636364,
        "recall": 0.56,
        "f1": 0.5957446808510

robustness_score

{
 "injection": {
  "n": 60,
  "any_change": 16,
  "changes": {
   "relevance": 2,
   "target": 2,
   "direction": 5,
   "sarcasm": 3,
   "behavior": 0,
   "pnl": 2,
   "emotion_fine": 6,
   "crowd": 3,
   "commercial": 0,
   "emotion_valence": 4
  },
  "product_change": 7,
  "direction_changes": 5
 },
 "forgery": {
  "n": 60,
  "any_change": 55,
  "changes": {
   "relevance": 4,
   "target": 15,
   "direction": 46,
   "sarcasm": 0,
   "behavior": 17,
   "pnl": 2,
   "emotion_fine": 15,
   "crowd": 31,
   "commercial": 0,
   "emotion_valence": 13
  },
  "direction_changes": 46
 },
 "date": {
  "n": 60,
  "any_change": 1,
  "changes": {
   "relevance": 0,
   "target": 0,
   "direction": 0,
   "sarcasm": 0,
   "behavior": 0,
   "pnl": 0,
   "emotion_fine": 1,
   "crowd": 0,
   "commercial": 0,
   "emotion_valence": 0
  },
  "direction_changes": 0
 },
 "swap": {
  "n": 60,
  "any_change": 6,
  "changes": {
   "relevance": 0,
   "target": 4,
   "direction": 0,
   "sarcasm": 0,
   "behavior": 0,
   "pnl": 0,
   "emotion_fine": 1,
   "crowd": 1,
   "commercial": 0,
   "emotion_valence": 0
  },
  "body_names_no_stock": {
   "n": 58,
   "target_changes": 3
  },
  "body_names_stock": {
   "n": 2,
   "target_changes": 1
  },
  "direction_changes": 0
 }
}

test_predict

{
 "rows": 940,
 "seconds": 15.72,
 "encode_seconds": 0.15,
 "posts_per_second": 59.8,
 "tokens_per_second": 8204,
 "tokens": 128957,
 "truncated_rows": 0,
 "which": "best",
 "max_prob_diff": 0.0,
 "argmax_identical": true,
 "deterministic": true,
 "max_memory_gb": 33.6
}

metrics_val

{
 "step": 1696,
 "selection_loss": 0.20436748314508657,
 "log_loss": {
  "relevance": 0.053753955979676846,
  "target": 0.1622685831315732,
  "direction": 0.30945366154198356,
  "sarcasm": 0.14681592124862222,
  "behavior": 0.13204823026110646,
  "pnl": 0.07633446204992973,
  "emotion_fine": 0.5265507831957529,
  "crowd": 0.2645692063593488,
  "commercial": 0.015189351947714416,
  "emotion_valence": 0.35669067573515767
 },
 "accuracy": {
  "relevance": 0.9797611147976112,
  "target": 0.9387469373468673,
  "direction": 0.8879829847571783,
  "sarcasm": 0.9527476373818691,
  "behavior": 0.96044802240112,
  "pnl": 0.9737486874343717,
  "emotion_fine": 0.8088904445222261,
  "crowd": 0.9068953447672383,
  "commercial": 0.9946914399469144,
  "emotion_valence": 0.8676933846692335
 },
 "rows": 3014,
 "eval_rows": 2237,
 "screen_primary": 0.12099744598139399
}

screen

{
 "winner": "B-last",
 "beats_default": [],
 "scores": {
  "B-last": {
   "primary": 0.1703279423207186,
   "secondary": 0.7503783758084439,
   "eval_rows": 2237,
   "run": "/workspace/runs/screen-B-last"
  },
  "I-last": {
   "primary": 0.1879070587120423,
   "secondary": 0.6152050313786558,
   "eval_rows": 2237,
   "run": "/workspace/runs/screen-I-last"
  },
  "B-mean": {
   "primary": 0.17661745790850816,
   "secondary": 0.6749563706026908,
   "eval_rows": 2237,
   "run": "/workspace/runs/screen-B-mean"
  },
  "I-mean": {
   "primary": 0.20586273438668987,
   "secondary": 0.5690600738310523,
   "eval_rows": 2237,
   "run": "/workspace/runs/screen-I-mean"
  }
 }
}

Run and acceptance

Production run (classifier-plan §4.4): arm B-last (google/gemma-4-12B base, last-token pooling) chosen by the screen (screen.json), trained on all 27,118 posts of the nsb-v2 training split (bundle v20261019-1009) for 2 epochs (1,696 steps); best checkpoint = last step (val selection loss 0.2044). temperatures.json is fitted on validation predictions of the reloaded checkpoint. Test = frozen hold-out holdout-1 (940 posts).

Acceptance (§5.5): no-go for v1 (REPORT.md): kappa below the bar on relevance, direction and sarcasm; aggregate parity fails on 13 thin test cells; bearish and valence shares 2-3 points off the teacher. Calibration ECE, fairness, drift, determinism and robustness pass; serving parity is not measured yet. Not for production use.