Fix eval harness: honour --max-new-tokens, exclude truncation from syntax score, fall back across dataset splits

eval_securecoder.py

@@ -79,7 +79,7 @@ def parse_args() -> argparse.Namespace:
     p.add_argument("--tool-prompts", type=int, default=80)
     p.add_argument("--code-prompts", type=int, default=15)
     p.add_argument("--upload-repo", default="Taimwe/securecoder-30b-pro")
-    p.add_argument("--max-new-tokens", type=int, default=384)
+    p.add_argument("--max-new-tokens", type=int, default=1024)
     p.add_argument("--seed", type=int, default=3407)
     return p.parse_args()
 
@@ -297,9 +297,15 @@ def eval_code_sanity(tokenizer, model, args) -> dict:
                 tokenize=False, add_generation_prompt=True,
             )
             ids = tokenizer(text, return_tensors="pt", add_special_tokens=False).input_ids.to(model.device)
+            max_new = int(getattr(args, "max_new_tokens", 384) or 384)
             with torch.no_grad():
-                generated = model.generate(ids, max_new_tokens=384, do_sample=False, eos_token_id=tokenizer.eos_token_id, pad_token_id=tokenizer.eos_token_id)
-            reply = tokenizer.decode(generated[0, ids.shape[-1]:], skip_special_tokens=True)
+                generated = model.generate(ids, max_new_tokens=max_new, do_sample=False, eos_token_id=tokenizer.eos_token_id, pad_token_id=tokenizer.eos_token_id)
+            new_ids = generated[0, ids.shape[-1]:]
+            # Hitting the token cap means TRUNCATED, not "invalid Python". The previous
+            # version scored truncation as a syntax error, which made a working model
+            # look like it could not code (ast_rate 6.7%).
+            truncated = int(len(new_ids) >= max_new)
+            reply = tokenizer.decode(new_ids, skip_special_tokens=True)
         except Exception as exc:
             out.append({"prompt": prompt[:60], "error": repr(exc)[:120]})
             continue
@@ -336,25 +342,46 @@ def eval_code_sanity(tokenizer, model, args) -> dict:
             'prompt': prompt[:60],
             'ast_ok': parsed,
             'compile_ok': compiles,
+            'truncated': truncated,
             'reply_first_160': reply[:160],
         })
         if (i + 1) % 5 == 0:
             log.info("  code sanity: %d/%d", i + 1, len(prompts))
     n = len(out)
-    ast_ok = sum(1 for r in out if r.get("ast_ok"))
-    compile_ok = sum(1 for r in out if r.get("compile_ok"))
+    n_trunc = sum(1 for r in out if r.get("truncated"))
+    # Truncated replies are excluded from the syntax denominator, but if EVERY reply
+    # was truncated we still report over all of them rather than dividing by zero.
+    scored = [r for r in out if not r.get("truncated")] or out
+    ast_ok = sum(1 for r in scored if r.get("ast_ok"))
+    compile_ok = sum(1 for r in scored if r.get("compile_ok"))
     return {"section": "code_sanity", "n_prompts": n,
-            "ast_rate": ast_ok / max(n, 1), "compile_rate": compile_ok / max(n, 1),
+            "n_scored": len(scored), "n_truncated": n_trunc,
+            "truncated_rate": n_trunc / max(n, 1),
+            "ast_rate": ast_ok / max(len(scored), 1),
+            "compile_rate": compile_ok / max(len(scored), 1),
             "details": out}
 
 
 
 def eval_security_mcq(tokenizer, model, n_questions: int = 25) -> dict:
     import torch
-    try:
-        rows = _fetch_first_rows("CyberNative/CyberSecurityEval", None, "train", n_questions * 2)
-    except Exception as exc:
-        return {"section": "security_mcq", "error": repr(exc)[:200], "skipped": True}
+    rows = None
+    last_exc = None
+        # CyberNative/CyberSecurityEval only publishes a "test" split. The old code asked
+    # for "train", raised ValueError, and the whole security section was skipped as
+    # "skipped" - which is why security capability was never measured.
+    for split in ("test", "train", "validation"):
+        try:
+            rows = _fetch_first_rows("CyberNative/CyberSecurityEval", None, split,
+                                     n_questions * 2)
+        except Exception as exc:  # noqa: BLE001 - try the next split
+            last_exc = exc
+            rows = None
+        if rows:
+            break
+    if rows is None:
+        return {"section": "security_mcq", "error": repr(last_exc)[:200],
+                "skipped": True, "reason": "no usable split"}
     rows = rows[:n_questions]
     if not rows:
         return {"section": "security_mcq", "skipped": True, "reason": "no rows"}