Fix: repair JSON-escaped newlines in training data (v1/v2 scored 0% ast_rate vs base 93%)

train_securecoder.py

@@ -582,6 +582,29 @@ def render_record(tokenizer, messages: list[dict], tools: list[dict] | None = No
 
 
 
+def _unescape_if_needed(text: str) -> tuple[str, bool]:
+    """Repair JSON-escaped newlines before training.
+
+    Parts of the data mix store message text already JSON-escaped, so the rendered
+    training text carries the two-character sequence backslash+n instead of a real
+    newline. Fine-tuning on that teaches the model to emit a literal "\\n", which is
+    why both v1 and v2 scored 0% ast_rate while the untouched base model scored 93%.
+
+    Only fires when escaped newlines clearly dominate, so ordinary code samples that
+    legitimately contain a backslash are left untouched.
+    """
+    literal = text.count("\\n")
+    if literal == 0:
+        return text, False
+    if text.count("\n") > literal:
+        return text, False
+    fixed = (text.replace("\\r\\n", "\n")
+                .replace("\\n", "\n")
+                .replace("\\t", "\t")
+                .replace('\\"', '"'))
+    return fixed, True
+
+
 def build_dataset(tokenizer, sources: list[Source], token: str | None, validate: bool):
     """Returns (records, stats). ``records`` are {"text", "source"} dicts."""
     records: list[dict] = []
@@ -589,7 +612,7 @@ def build_dataset(tokenizer, sources: list[Source], token: str | None, validate:
 
     for src in sources:
         entry = {"source": source_name(src), "note": src.note, "kept": 0, "skipped": 0,
-                 "tool_samples": 0, "chars": 0, "error": None}
+                 "tool_samples": 0, "chars": 0, "repaired": 0, "error": None}
         try:
             rows = load_source(src, token, progress=validate)
             for row in rows:
@@ -603,6 +626,7 @@ def build_dataset(tokenizer, sources: list[Source], token: str | None, validate:
                     continue
                 try:
                     text = render_record(tokenizer, messages, tools)
+                    text, repaired = _unescape_if_needed(text)
                 except Exception as exc:  # noqa: BLE001 - bad template input, skip row
                     if entry["skipped"] < 3:
                         log.warning("    render failed (%s): %s", source_name(src), exc)
@@ -614,6 +638,8 @@ def build_dataset(tokenizer, sources: list[Source], token: str | None, validate:
                 records.append({"text": text, "source": source_name(src)})
                 entry["kept"] += 1
                 entry["chars"] += len(text)
+                if repaired:
+                    entry["repaired"] += 1
                 if tools:
                     entry["tool_samples"] += 1
         except Exception as exc:  # noqa: BLE001 - one bad dataset must not kill the run
@@ -621,8 +647,9 @@ def build_dataset(tokenizer, sources: list[Source], token: str | None, validate:
             log.error("  %s failed: %s", source_name(src), exc)
 
         stats.append(entry)
-        log.info("  %-58s kept=%-6d skipped=%-5d tools=%-5d",
-                 entry["source"], entry["kept"], entry["skipped"], entry["tool_samples"])
+        log.info("  %-58s kept=%-6d skipped=%-5d tools=%-5d repaired=%-5d",
+                 entry["source"], entry["kept"], entry["skipped"], entry["tool_samples"],
+                 entry.get("repaired", 0))
 
     random.shuffle(records)
     return records, stats