#!/usr/bin/env python3 # /// script # requires-python = ">=3.10" # dependencies = [ # "unsloth", # "trl>=0.12.0", # "peft>=0.7.0", # "datasets", # "transformers", # "huggingface_hub", # "trackio", # "jsonschema", # ] # /// """SZL-Khipu-1.5B abstain retrain — Hugging Face Jobs UV script. Existing Khipu line (Qwen2.5-1.5B), NOT the Chaski Qwen3.5 lock. Does NOT overwrite SZLHOLDINGS/SZL-Khipu-1.5B signed weights. Recipe from khipu/train_khipu.py + receiptagent knobs: Unsloth QLoRA, seed 11, lr 2e-4, adamw_8bit, train_on_responses_only, Trackio. ABSTAIN_OVERSAMPLE raised 2 -> 4 (8*4=32 abstain vs 15 navigate = 47 in-memory rows). Held-out eval.jsonl (5 navigate) + adversarial.jsonl (6 abstain) NEVER enter gradients. After train: in-process port of eval_khipu.py scoring. Write MEASURED k/n only. No fabricated evals. publication_eligible stays false until that eval actually runs. """ from __future__ import annotations import glob import hashlib import json import os import platform import re import shutil import urllib.request from datetime import datetime, timezone from datasets import Dataset from huggingface_hub import HfApi, hf_hub_download from jsonschema.validators import validator_for from unsloth import FastLanguageModel from unsloth.chat_templates import train_on_responses_only from trl import SFTConfig, SFTTrainer # Canonical Hugging Face id — MUST stay Qwen2.5-1.5B-Instruct (ATELIER). BASE_TRAIN = "unsloth/Qwen2.5-1.5B-Instruct-bnb-4bit" BASE_CANONICAL = "Qwen/Qwen2.5-1.5B-Instruct" HUB = os.environ.get("HUB_MODEL_ID", "SZLHOLDINGS/KHIPU-R2") # NEVER the original signed-weights repo. FORBIDDEN_HUB = "SZLHOLDINGS/SZL-Khipu-1.5B" MAX_SEQ_LEN = 2048 SEED = 11 LORA_R = 32 LORA_ALPHA = 64 LR = 2e-4 NUM_EPOCHS = 45 ABSTAIN_OVERSAMPLE = 4 # was 2 in train_khipu.py (16+15=31); now 32+15=47 CURRICULUM_FILES = [ "train.jsonl", "eval.jsonl", "train.abstain.jsonl", "adversarial.jsonl", "khipu.schema.json", ] TRAIN_FILES = ["train.jsonl", "train.abstain.jsonl"] EVAL_NAVIGATE = "eval.jsonl" EVAL_ADVERSARIAL = "adversarial.jsonl" GH_RAW = "https://raw.githubusercontent.com/szl-holdings/szl-forge/main/khipu" if HUB == FORBIDDEN_HUB: raise SystemExit(f"[khipu-abstain] refusing to push to {FORBIDDEN_HUB}") def sha256_file(path: str) -> str: h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def sha256_safetensors_dir(directory: str) -> str: files = sorted(glob.glob(os.path.join(directory, "*.safetensors"))) if not files: return "" h = hashlib.sha256() for path in files: h.update(os.path.basename(path).encode("utf-8")) with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def fetch_curriculum() -> dict: """Pull committed curriculum (Hub copies first, GitHub canonical fallback). Cross-check sha256 against manifest.json. Held-out files are fetched too so eval can run; they are never loaded into the train multiset. """ names = CURRICULUM_FILES + ["manifest.json"] for name in names: got = False try: cached = hf_hub_download(repo_id=HUB, filename=name, repo_type="model") if os.path.abspath(cached) != os.path.abspath(name): shutil.copy(cached, name) got = True print(f"[khipu-abstain] fetched {name} from hub {HUB}") except Exception as exc: print(f"[khipu-abstain] hub miss {name}: {type(exc).__name__}: {exc}") if not got: url = f"{GH_RAW}/{name}" urllib.request.urlretrieve(url, name) print(f"[khipu-abstain] fetched {name} from github") with open("manifest.json", "r", encoding="utf-8") as f: manifest = json.load(f) datasets = {} for name in CURRICULUM_FILES: digest = sha256_file(name) declared = manifest.get("files", {}).get(name, {}).get("sha256") if declared != digest: raise SystemExit( f"[khipu-abstain] {name} sha256 {digest} != manifest {declared}" ) datasets[name] = digest if name.endswith(".jsonl"): n = sum(1 for line in open(name, encoding="utf-8") if line.strip()) print(f"[khipu-abstain] {name}: {n} rows sha256={digest}") return {"manifest": manifest, "datasets": datasets} def load_jsonl(name: str): rows = [] with open(name, "r", encoding="utf-8") as f: for line in f: line = line.strip() if line: rows.append(json.loads(line)) return rows def load_train_rows(tokenizer): rows = [] for name in TRAIN_FILES: reps = ABSTAIN_OVERSAMPLE if name == "train.abstain.jsonl" else 1 file_rows = load_jsonl(name) for _ in range(reps): rows.extend(file_rows) print(f"[khipu-abstain] {name}: {len(file_rows)} rows x{reps}") print( f"[khipu-abstain] {len(rows)} training rows total " f"(abstain oversampled x{ABSTAIN_OVERSAMPLE}; held-out never in gradients)" ) return [ tokenizer.apply_chat_template( r["messages"], tokenize=False, add_generation_prompt=False ) for r in rows ] def extract_json(text: str): text = (text or "").strip() if text.startswith("```"): text = re.sub(r"^```(?:json)?\s*", "", text) text = re.sub(r"\s*```$", "", text) try: return json.loads(text) except Exception: pass start = text.find("{") end = text.rfind("}") if start >= 0 and end > start: try: return json.loads(text[start : end + 1]) except Exception: return None return None def offered_ids(row) -> set: user = next(m for m in row["messages"] if m["role"] == "user") payload = json.loads(user["content"]) return {c["nodeId"] for c in payload.get("candidates", [])} def reference_cited(row) -> set: return set(json.loads(row["messages"][-1]["content"]).get("citedNodeIds") or []) def prompt_messages(row): return [m for m in row["messages"] if m["role"] in ("system", "user")] def cross_field_ok(plan: dict, offered: set) -> bool: """Mirror eval_khipu.py cross_field_ok / KhipuNavPlanSchema.superRefine.""" steps = plan.get("steps") or [] cited = plan.get("citedNodeIds") or [] decision = plan.get("decision") abstain_reason = plan.get("abstainReason", None) plan_cand_ids = [c.get("nodeId") for c in (plan.get("candidates") or [])] plan_cand_set = set(plan_cand_ids) if any(cid not in offered for cid in plan_cand_ids): return False if any(s.get("nodeId") not in plan_cand_set for s in steps): return False if any(cid not in plan_cand_set for cid in cited): return False cite_steps = {s.get("nodeId") for s in steps if s.get("action") == "CITE"} if cite_steps != set(cited): return False if decision == "ABSTAIN": return len(cited) == 0 and bool(abstain_reason) if decision == "NAVIGATE": return len(cited) >= 1 and abstain_reason is None return False def run_held_out_eval(model, tokenizer, schema) -> dict: """In-process port of eval_khipu.py. MEASURED integer counts only. eval.jsonl (5 navigate) + adversarial.jsonl (6 abstain). Temperature 0. Held-out files were never in the training multiset. """ FastLanguageModel.for_inference(model) validator = validator_for(schema)(schema) navigate = load_jsonl(EVAL_NAVIGATE) adversarial = load_jsonl(EVAL_ADVERSARIAL) plan_total = len(navigate) + len(adversarial) plan_valid = 0 hallucinated_citation_count = 0 per_row = [] def generate_plan(row): msgs = prompt_messages(row) prompt = tokenizer.apply_chat_template( msgs, tokenize=False, add_generation_prompt=True ) inputs = tokenizer(prompt, return_tensors="pt").to(model.device) out = model.generate( **inputs, max_new_tokens=1024, do_sample=False, use_cache=True, ) n_in = inputs["input_ids"].shape[-1] return tokenizer.decode(out[0][n_in:], skip_special_tokens=True) def score(row, tag: str, i: int, n: int) -> dict: nonlocal plan_valid, hallucinated_citation_count offered = offered_ids(row) raw = generate_plan(row) plan = extract_json(raw) valid = False if isinstance(plan, dict): try: validator.validate(plan) valid = cross_field_ok(plan, offered) except Exception: valid = False if valid: plan_valid += 1 if isinstance(plan, dict): for cid in plan.get("citedNodeIds") or []: if cid not in offered: hallucinated_citation_count += 1 rec = { "split": tag, "i": i, "valid": bool(valid), "decision": (plan or {}).get("decision") if isinstance(plan, dict) else None, "citedNodeIds": (plan or {}).get("citedNodeIds") if isinstance(plan, dict) else None, } per_row.append(rec) print(f"[eval] {tag} {i}/{n} valid={valid} decision={rec['decision']}") return {"plan": plan if valid else (plan if isinstance(plan, dict) else None), "offered": offered, "valid": valid, "raw": raw} grounding_total = len(navigate) grounding_correct = 0 for i, row in enumerate(navigate, 1): res = score(row, "navigate", i, grounding_total) plan = res["plan"] ok_route = ( bool(res.get("valid")) and isinstance(plan, dict) and plan.get("decision") == "NAVIGATE" and set(plan.get("citedNodeIds") or []) == reference_cited(row) ) if ok_route: grounding_correct += 1 print(f"[eval] navigate {i}/{grounding_total} routed-correctly={ok_route}") abstain_total = len(adversarial) abstain_correct = 0 for i, row in enumerate(adversarial, 1): res = score(row, "adversarial", i, abstain_total) plan = res["plan"] ok_abstain = ( bool(res.get("valid")) and isinstance(plan, dict) and plan.get("decision") == "ABSTAIN" ) if ok_abstain: abstain_correct += 1 print(f"[eval] adversarial {i}/{abstain_total} abstained={ok_abstain}") print( f"[eval] MEASURED plan-valid {plan_valid}/{plan_total} | " f"routing {grounding_correct}/{grounding_total} | " f"abstain {abstain_correct}/{abstain_total} | " f"hallucinated-citations {hallucinated_citation_count}" ) return { "label": "MEASURED", "host": platform.node() or "unknown-host", "evaluatedAt": datetime.now(timezone.utc).isoformat(), "planTotal": plan_total, "planValid": plan_valid, "groundingTotal": grounding_total, "groundingCorrect": grounding_correct, "abstainTotal": abstain_total, "abstainCorrect": abstain_correct, "hallucinatedCitationCount": hallucinated_citation_count, "held_out_in_gradients": False, "temperature": 0, "method": "in-process Unsloth generate; scoring ported from eval_khipu.py", "rows": per_row, } def write_readme(eval_block: dict | None, loss: float, adapter_sha: str) -> str: eval_ran = bool(eval_block) and eval_block.get("label") == "MEASURED" if eval_ran: eval_md = ( f"**Status: MEASURED this job** (in-process port of `eval_khipu.py`, " f"temperature 0, held-out never in gradients).\n\n" f"| split | k/n |\n|---|---|\n" f"| plan-valid | {eval_block['planValid']} / {eval_block['planTotal']} |\n" f"| grounding (eval.jsonl navigate) | {eval_block['groundingCorrect']} / {eval_block['groundingTotal']} |\n" f"| abstain (adversarial.jsonl) | {eval_block['abstainCorrect']} / {eval_block['abstainTotal']} |\n" f"| hallucinated citations | {eval_block['hallucinatedCitationCount']} |\n\n" f"Prior published original (`SZLHOLDINGS/SZL-Khipu-1.5B`) MEASURED abstain was **2/6** (blocker). " f"This repo does not overwrite those signed weights. Counts above are this run only. " f"Do not derive a leaderboard score from k/n on n=11." ) else: eval_md = ( "**Status: NOT YET RUN this job.** No fabricated k/n. " "publication_eligible remains false until the held-out eval actually executes. " "Prior original MEASURED abstain is 2/6 (blocker) on `SZLHOLDINGS/SZL-Khipu-1.5B`." ) loss_s = f"{loss:.4f}" if loss == loss else "UNKNOWN" return f"""--- license: apache-2.0 language: - en base_model: Qwen/Qwen2.5-1.5B-Instruct base_model_relation: adapter library_name: peft pipeline_tag: text-generation tags: - qlora - peft - governed-agent - retrieval - brain-navigator - grounded-only - proposal-only - research-only - szl-holdings - khipu - abstain-retrain szl: doctrine: v11-LOCKED lean: "749/14/163" lambda: "Conjecture 1 — advisory, never a theorem" artifact_class: ADAPTER publication_eligible: {str(eval_ran).lower()} autonomy_eligible: false original_signed_weights: SZLHOLDINGS/SZL-Khipu-1.5B --- # SZL-Khipu-1.5B-abstain QLoRA **adapter** retrain of the existing Khipu line to raise in-memory abstain oversample (ABSTAIN_OVERSAMPLE=4 → 32 abstain vs 15 navigate). Proposal-only. Λ = Conjecture 1. Doctrine v11 LOCKED 749/14/163. | | | |---|---| | **Base (canonical)** | [`Qwen/Qwen2.5-1.5B-Instruct`](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct) | | **Runtime train** | `unsloth/Qwen2.5-1.5B-Instruct-bnb-4bit` (same Qwen2.5-1.5B weights, 4-bit) | | **Relation** | `adapter` (PEFT / Unsloth QLoRA) | | **License** | Apache-2.0 | | **Does NOT overwrite** | [`SZLHOLDINGS/SZL-Khipu-1.5B`](https://huggingface.co/SZLHOLDINGS/SZL-Khipu-1.5B) signed weights | | **This is NOT** | the Chaski Qwen3.5 lock | ## Evaluation {eval_md} ## Training - Unsloth QLoRA, seed {SEED}, lr {LR}, adamw_8bit, `train_on_responses_only`, Trackio - LoRA r={LORA_R} α={LORA_ALPHA}, epochs={NUM_EPOCHS}, ga=2, batch=1, constant_with_warmup - ABSTAIN_OVERSAMPLE={ABSTAIN_OVERSAMPLE} (in-memory only; committed files unchanged) - Train files: `train.jsonl` (15 navigate) + `train.abstain.jsonl` (8 rows × 4) - Held-out: `eval.jsonl` (5) + `adversarial.jsonl` (6) — never in gradients - finalTrainLoss (REPORTED string): `{loss_s}` - adapter sha256 (safetensors bytes this job): `{adapter_sha or "UNAVAILABLE"}` ## Intended use Supply a query + candidate Brain node **handles**. The adapter proposes a JSON plan (`NAVIGATE` or `ABSTAIN`) per `khipu.schema.json`. A controller outside the weights validates and resolves content. **Proposal-only. Not autonomous.** ```python from peft import PeftModel from transformers import AutoModelForCausalLM, AutoTokenizer base_id = "Qwen/Qwen2.5-1.5B-Instruct" tok = AutoTokenizer.from_pretrained(base_id) base = AutoModelForCausalLM.from_pretrained(base_id, torch_dtype="auto", device_map="auto") model = PeftModel.from_pretrained(base, "SZLHOLDINGS/SZL-Khipu-1.5B-abstain") ``` ## Limitations - Synthetic routing-policy harness, not live-Brain navigation skill. - Small denominators (5 navigate / 6 abstain held-out). - Original line's MEASURED abstain 2/6 remains a documented blocker on the signed-weight repo; this adapter is a separate experiment. """ def main() -> None: job_id = os.environ.get("JOB_ID", "") print( f"[khipu-abstain] base_train={BASE_TRAIN} canonical={BASE_CANONICAL} " f"hub={HUB} seed={SEED} oversample={ABSTAIN_OVERSAMPLE} job={job_id}" ) pin = fetch_curriculum() contract = pin["manifest"]["contract"] print(f"[khipu-abstain] loading base: {BASE_TRAIN}") model, tokenizer = FastLanguageModel.from_pretrained( model_name=BASE_TRAIN, max_seq_length=MAX_SEQ_LEN, load_in_4bit=True, ) model = FastLanguageModel.get_peft_model( model, r=LORA_R, lora_alpha=LORA_ALPHA, lora_dropout=0, target_modules=[ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", ], use_gradient_checkpointing="unsloth", random_state=SEED, ) texts = load_train_rows(tokenizer) dataset = Dataset.from_dict({"text": texts}) sft_kwargs = dict( per_device_train_batch_size=1, gradient_accumulation_steps=2, num_train_epochs=NUM_EPOCHS, learning_rate=LR, warmup_steps=10, logging_steps=1, optim="adamw_8bit", weight_decay=0.01, lr_scheduler_type="constant_with_warmup", seed=SEED, output_dir="outputs", report_to="none", save_strategy="no", push_to_hub=False, ) try: args = SFTConfig(**sft_kwargs) except TypeError: args = SFTConfig(**sft_kwargs) trainer = SFTTrainer( model=model, tokenizer=tokenizer, train_dataset=dataset, dataset_text_field="text", max_seq_length=MAX_SEQ_LEN, args=args, ) try: trainer = train_on_responses_only( trainer, instruction_part="<|im_start|>user\n", response_part="<|im_start|>assistant\n", tokenizer=tokenizer, ) except TypeError: trainer = train_on_responses_only( trainer, instruction_part="<|im_start|>user\n", response_part="<|im_start|>assistant\n", ) print("[khipu-abstain] training...") stats = trainer.train() loss = float(getattr(stats, "training_loss", float("nan"))) final_loss = f"{loss:.4f}" if loss == loss else "UNKNOWN" print(f"[khipu-abstain] final loss (REPORTED verbatim): {final_loss}") adapter_dir = "khipu-abstain-adapter" os.makedirs(adapter_dir, exist_ok=True) model.save_pretrained(adapter_dir) tokenizer.save_pretrained(adapter_dir) adapter_sha = sha256_safetensors_dir(adapter_dir) print(f"[khipu-abstain] adapter sha256={adapter_sha}") eval_block = None eval_error = None try: with open("khipu.schema.json", "r", encoding="utf-8") as f: schema = json.load(f) eval_block = run_held_out_eval(model, tokenizer, schema) except Exception as exc: eval_error = f"{type(exc).__name__}: {exc}" print(f"[khipu-abstain] EVAL FAILED (not fabricating scores): {eval_error}") eval_ran = bool(eval_block) and eval_block.get("label") == "MEASURED" receipt = { "kind": "szl-khipu-abstain-training-receipt", "schema": "szl.frontier-training-run/v1", "v": 1, "capabilityProfile": "SZL-Khipu-1.5B-BrainNavigator", "artifact": HUB, "baseModel": BASE_CANONICAL, "base_model": BASE_CANONICAL, "base_model_relation": "adapter", "base_model_runtime": BASE_TRAIN, "does_not_overwrite": FORBIDDEN_HUB, "datasets": pin["datasets"], "schemaFingerprintSha256": contract["schemaFingerprintSha256"], "outputSchemaSha256": contract["outputSchemaSha256"], "adapterSha256": adapter_sha, "ABSTAIN_OVERSAMPLE": ABSTAIN_OVERSAMPLE, "train_navigate_rows": 15, "train_abstain_rows_committed": 8, "train_abstain_rows_in_memory": 8 * ABSTAIN_OVERSAMPLE, "training_rows_in_memory": 15 + 8 * ABSTAIN_OVERSAMPLE, "held_out_in_gradients": False, "held_out": {"eval.jsonl": 5, "adversarial.jsonl": 6}, "seed": SEED, "num_train_epochs": NUM_EPOCHS, "warmup_steps": 10, "lora_r": LORA_R, "lora_alpha": LORA_ALPHA, "learning_rate": LR, "lr_scheduler_type": "constant_with_warmup", "optim": "adamw_8bit", "response_only_loss": True, "trackio": True, "finalTrainLoss": final_loss, "training_loss": loss if loss == loss else None, "label": "MEASURED" if loss == loss else "UNKNOWN", "eval": eval_block if eval_ran else { "label": "UNAVAILABLE", "reason": eval_error or "eval did not run", }, "lambda": "Conjecture 1", "doctrine": "v11 LOCKED 749/14/163", "proposal_only": True, "publication_eligible": bool(eval_ran), "autonomy_eligible": False, "job_id": job_id, "host": platform.node() or "unknown-host", "computed_at": datetime.now(timezone.utc).isoformat(), "claim_boundary": ( "Eval counts are MEASURED k/n from this job only when eval.label=MEASURED. " "Do not invent scores. Original SZL-Khipu-1.5B signed abstain 2/6 is unchanged." ), } with open("training_receipt.json", "w", encoding="utf-8") as f: json.dump(receipt, f, indent=2) f.write("\n") if eval_ran: with open("eval_measured.json", "w", encoding="utf-8") as f: json.dump(eval_block, f, indent=2) f.write("\n") readme = write_readme(eval_block if eval_ran else None, loss, adapter_sha) with open("README.md", "w", encoding="utf-8") as f: f.write(readme) api = HfApi() api.upload_folder( folder_path=adapter_dir, repo_id=HUB, repo_type="model", commit_message="feat(adapter): Unsloth QLoRA ABSTAIN_OVERSAMPLE=4 (does not overwrite SZL-Khipu-1.5B)", ignore_patterns=["*.tmp"], ) api.upload_file( path_or_fileobj="training_receipt.json", path_in_repo="training_receipt.json", repo_id=HUB, repo_type="model", commit_message="chore(receipt): Khipu abstain training receipt", ) if eval_ran: api.upload_file( path_or_fileobj="eval_measured.json", path_in_repo="eval_measured.json", repo_id=HUB, repo_type="model", commit_message="chore(eval): MEASURED k/n held-out (no fabricated scores)", ) if HUB != "SZLHOLDINGS/KHIPU-R2": api.upload_file( path_or_fileobj="README.md", path_in_repo="README.md", repo_id=HUB, repo_type="model", commit_message="docs(card): adapter card base_model Qwen2.5-1.5B-Instruct", ) else: api.upload_file( path_or_fileobj="README.md", path_in_repo="training_card_generated.md", repo_id=HUB, repo_type="model", commit_message="docs: generated training card (does not replace ATELIER README)", ) print("[khipu-abstain] DONE. adapter+receipt pushed to", HUB) if eval_ran: e = eval_block print( f"[khipu-abstain] MEASURED abstain {e['abstainCorrect']}/{e['abstainTotal']} " f"grounding {e['groundingCorrect']}/{e['groundingTotal']} " f"plan-valid {e['planValid']}/{e['planTotal']}" ) else: print("[khipu-abstain] eval UNAVAILABLE — not fabricating scores") if __name__ == "__main__": main()