Add best scaled combo A080 candidate

candidates/best-scaled-combo-a080/README.md

@@ -0,0 +1,49 @@
+---
+language:
+- en
+pipeline_tag: text-generation
+tags:
+- testgeniy
+- causal-lm
+- reasoning
+- mathematics
+- logic
+- dialogue
+- dpo
+- lora
+- 4k-context
+- small-language-model
+---
+
+# TestGeniy Best Scaled Combo A080
+
+A 488M-parameter TestGeniy research checkpoint optimized for mathematical reasoning, logic, and dialogue retention at a 4096-token context window.
+
+## Training
+
+This candidate combines two independently trained low-rank task vectors: a balanced math/logic/dialogue adapter and an advanced-math adapter. The advanced vector is merged at 0.5, then the combined update is scaled to 0.8 from the selected SFT anchor.
+
+Training data includes verifier-filtered augmented problems from `nvidia/OpenMathInstruct-2`, concise `open-r1/OpenR1-Math-220k` solutions, and replay from StrategyQA, ARC-Challenge train, FOLIO train, OASST1, and HelpSteer2. Evaluation questions were excluded by normalized exact matching and shared 12-token-window filtering. No held-out benchmark rows were used for training.
+
+## Expanded paired gate
+
+Greedy user-only decoding with fixed seed 20260819.
+
+| Benchmark | Selected SFT | This candidate |
+|---|---:|---:|
+| GSM8K | 4/30 | 6/30 |
+| MATH-500 | 2/15 | 3/15 |
+| ARC-Challenge | 4/30 | 8/30 |
+| FOLIO | 12/30 | 12/30 |
+| HelpSteer2 pairwise | 48.0% | 47.5% |
+| Composite | 25.60% | 30.83% |
+
+OASST1 validation response NLL improved from 3.04000 to 3.03916 on 100 fixed English human direct-response pairs.
+
+## Architecture
+
+24 transformer layers, hidden size 1280, 10 attention heads, 2 KV heads, SwiGLU FFN size 3584, tied embeddings, 4K context, RoPE theta 500000, local attention block 1024, and full NoPE attention in layers 3, 7, 11, 15, 19, and 23.
+
+## Limitations
+
+This is a small experimental model. It remains weak on difficult mathematics and can produce incorrect or repetitive reasoning. The reported gate is an expanded regression gate, not a full benchmark leaderboard result.

candidates/best-scaled-combo-a080/config.json

@@ -0,0 +1,44 @@
+{
+  "architectures": [
+    "TestGeniyForCausalLM"
+  ],
+  "model_type": "testgeniy",
+  "vocab_size": 49216,
+  "hidden_size": 1280,
+  "num_hidden_layers": 24,
+  "num_attention_heads": 10,
+  "num_key_value_heads": 2,
+  "intermediate_size": 3584,
+  "head_dim": 128,
+  "max_position_embeddings": 4096,
+  "rope_theta": 500000.0,
+  "local_block_size": 1024,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "base_checkpoint": "best_scaled_combo_a080",
+  "weights_updated": true
+}

candidates/best-scaled-combo-a080/manifest.json

@@ -0,0 +1,68 @@
+{
+  "release": "best_scaled_combo_a080",
+  "base": "sft_pilot_lr7e7_step020",
+  "context_length": 4096,
+  "weights_updated": true,
+  "benchmark_rows_used": false,
+  "training": {
+    "balanced_lora": {
+      "rank": 8,
+      "alpha": 16,
+      "step": 50
+    },
+    "advanced_lora": {
+      "rank": 16,
+      "alpha": 32,
+      "step": 50,
+      "merge_alpha": 0.5
+    },
+    "global_task_vector_scale": 0.8
+  },
+  "data": {
+    "openmath2": "1500 augmented_gsm8k + 1000 augmented_math, math_verify filtered",
+    "replay": "OpenR1-Math, StrategyQA, ARC train, FOLIO train, OASST1, HelpSteer2",
+    "decontamination": "exact normalized hash and shared 12-token window against held-out evaluation questions"
+  },
+  "gate": {
+    "protocol": {
+      "seed": 20260819,
+      "decoding": "greedy",
+      "prompt": "user-only",
+      "gsm8k_n": 30,
+      "math500_n": 15,
+      "arc_n": 30,
+      "folio_n": 30,
+      "helpsteer2_pairs": 200
+    },
+    "selected_sft": {
+      "gsm8k": 13.333333333333334,
+      "math500": 13.333333333333334,
+      "arc": 13.333333333333334,
+      "folio": 40.0,
+      "helpsteer2": 48.0,
+      "composite": 25.6
+    },
+    "best_scaled_combo": {
+      "gsm8k": 20.0,
+      "math500": 20.0,
+      "arc": 26.666666666666668,
+      "folio": 40.0,
+      "helpsteer2": 47.5,
+      "composite": 30.833333333333332
+    },
+    "checkpoint": "/marimo/testgeniy_data/best_scaled_combo_a080.pt",
+    "data": [
+      "/marimo/testgeniy_data/openmath2_verified_manifest.json",
+      "/marimo/testgeniy_data/sft_v1_manifest.json"
+    ],
+    "delta_composite": 5.233333333333331
+  },
+  "oasst1": {
+    "split": "OpenAssistant/oasst1:validation",
+    "selection": "English non-synthetic direct prompter-assistant rank 0 or unranked, sorted message_id, max_len 768",
+    "n": 100,
+    "selected_sft_response_nll": 3.040004561831332,
+    "best_scaled_response_nll": 3.0391626408923074,
+    "delta": -0.0008419209390246858
+  }
+}

candidates/best-scaled-combo-a080/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f57967d1020d9bd6fc4aea8924e81f73e1de6f88773948a708984350bfe63729
+size 1101624520

candidates/best-scaled-combo-a080/tokenizer_config.json

@@ -0,0 +1,21 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|begin|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|doc|>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>"
+  ],
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 1000000000000,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}