Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+LLMs/qwen3-4b-base-sft-qwen3-8b/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+data/dapo-math-17k/dapo-math-17k.jsonl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+.pytest_cache/
+*.egg-info/
+outputs/
+

LLMs/qwen3-4b-base-sft-qwen3-8b/README.md

@@ -0,0 +1,61 @@
+---
+library_name: transformers
+license: other
+base_model: model_weights/qwen3-4b-base
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: qwen3-4b-base-sft-qwen3-8b
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# qwen3-4b-base-sft-qwen3-8b
+
+This model is a fine-tuned version of [model_weights/qwen3-4b-base](https://huggingface.co/model_weights/qwen3-4b-base) on the openthoughts3_300k_qwen3-8b dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 8e-05
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- training_steps: 3000
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.52.4
+- Pytorch 2.12.0+cu130
+- Datasets 3.6.0
+- Tokenizers 0.21.1

LLMs/qwen3-4b-base-sft-qwen3-8b/added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

LLMs/qwen3-4b-base-sft-qwen3-8b/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.34443168771526983,
+    "total_flos": 3.429011909561549e+19,
+    "train_loss": 0.27564545996983847,
+    "train_runtime": 45365.0148,
+    "train_samples_per_second": 2.116,
+    "train_steps_per_second": 0.066
+}

LLMs/qwen3-4b-base-sft-qwen3-8b/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

LLMs/qwen3-4b-base-sft-qwen3-8b/config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 9728,
+  "max_position_embeddings": 40960,
+  "max_window_layers": 36,
+  "model_type": "qwen3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

LLMs/qwen3-4b-base-sft-qwen3-8b/generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "temperature": 0.6,
+  "top_k": 20,
+  "top_p": 0.95,
+  "transformers_version": "4.52.4"
+}

LLMs/qwen3-4b-base-sft-qwen3-8b/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00619761893181a1e70ec80551accdf1a17b173a4cc956de5d744e42239ddda5
+size 15190896

LLMs/qwen3-4b-base-sft-qwen3-8b/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:591aed33a9c8e68474a30fd166f755fa3a348f6d6c1c1ed5e1c49e329ff04051
+size 18464796

LLMs/qwen3-4b-base-sft-qwen3-8b/model.safetensors.index.json

@@ -0,0 +1,406 @@
+{
+  "metadata": {
+    "total_size": 8822848512
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.20.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

LLMs/qwen3-4b-base-sft-qwen3-8b/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

LLMs/qwen3-4b-base-sft-qwen3-8b/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

LLMs/qwen3-4b-base-sft-qwen3-8b/tokenizer_config.json

@@ -0,0 +1,240 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

LLMs/qwen3-4b-base-sft-qwen3-8b/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.34443168771526983,
+    "total_flos": 3.429011909561549e+19,
+    "train_loss": 0.27564545996983847,
+    "train_runtime": 45365.0148,
+    "train_samples_per_second": 2.116,
+    "train_steps_per_second": 0.066
+}

LLMs/qwen3-4b-base-sft-qwen3-8b/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b28ccff420c215e9d7eaa03851e14301ec0e1720a9cca5e39fab81383ec4e4a7
+size 7377

README.md

@@ -0,0 +1,118 @@
+# DAPO Math Jacobian matrices
+
+Fits averaged Jacobian transport matrices from Anthropic's Jacobian Lens
+method on a local Qwen checkpoint and DAPO Math prompts.
+
+For block output `l` and target block output `L`:
+
+```text
+J_l = E_prompt,source-position,future-target-position[d h_L / d h_l]
+```
+
+The estimator follows Anthropic's Apache-2.0
+[`jacobian-lens`](https://github.com/anthropics/jacobian-lens) reference code.
+
+## Layer convention
+
+Qwen3-4B has 36 transformer blocks indexed `0..35`. The default target is the
+output of block 35, and matrices are fitted for block outputs `0..34`. The
+target-to-itself map is exactly identity and is included as matrix 35, giving
+one exported matrix for every transformer-block output. The embedding boundary
+is not included in this first version.
+
+## Install and smoke fit
+
+```bash
+python -m pip install -e .
+fit-jacobians \
+  --model LLMs/qwen3-4b-base-sft-qwen3-8b \
+  --data data/dapo-math-17k/dapo-math-17k.jsonl \
+  --num-prompts 20 \
+  --dim-batch 8 \
+  --output-dir outputs/smoke-20
+```
+
+The resumable checkpoint keeps FP32 sums. `lens-bf16.pt` contains the final
+BF16 means. Increase `--dim-batch` only if GPU memory permits.
+
+For the main fit, change `--num-prompts` to `1000` and preferably set
+`--checkpoint-every 10`.
+
+## Multiple GPUs
+
+The multi-GPU launcher fits disjoint prompt shards with one complete model
+replica per GPU, then merges the FP32 sums exactly before converting to BF16:
+
+```bash
+GPUS=0,1,2,3 \
+NUM_PROMPTS=1000 \
+OUTPUT_DIR=outputs/main-1000 \
+bash scripts/fit_multi_gpu.sh
+```
+
+For two GPUs and the default 20-prompt smoke test:
+
+```bash
+GPUS=0,1 bash scripts/fit_multi_gpu.sh
+```
+
+Optional variables include `DIM_BATCH`, `MAX_SEQ_LEN`, `CHECKPOINT_EVERY`,
+`MODEL_PATH`, `DATA_PATH`, `SEED`, and `PYTHON_BIN`. Each process sees its
+assigned physical GPU as `cuda:0`. Failed shards retain their checkpoints, so
+running the same command resumes them rather than restarting.
+
+This first stage uses question text only. Prompt-plus-rollout windows require a
+separate frozen rollout corpus; adding that corpus is the next stage and does
+not change the Jacobian estimator.
+
+## Interactive sanity check
+
+Use held-out samples after the fitting range. For a lens fitted with the first
+20 seed-17 shuffled prompts:
+
+```bash
+python -m math_jlens.explore \
+  --model LLMs/qwen3-4b-base-sft-qwen3-8b \
+  --data data/dapo-math-17k/dapo-math-17k.jsonl \
+  --lens outputs/multi-gpu-smoke-20/lens-bf16.pt \
+  --fit-count 20 \
+  --seed 17 \
+  --sample-index 0 \
+  --device cuda:0
+```
+
+The explorer generates one deterministic solution, caches it, performs a
+teacher-forced residual capture, and opens a prompt. Useful commands:
+
+```text
+tokens [start] [end]          show token positions (P=prompt, R=response)
+inspect POSITION [TOP_K]      top J-lens tokens across every layer
+compare POSITION LAYER        compare J-lens against ordinary logit lens
+trace "TOKEN TEXT" [POSITION] show token ranks across layers
+save START END [LAYER|all] [TOP_K] [FILE]
+                              export original and J-lens tokens to text
+answer                        trace the first gold-answer token before output
+problem                       show the complete DAPO problem
+output                        show the complete generated solution
+result                        show extracted answer, ground truth, and match
+info                          show sample, problem, output, and result together
+sample INDEX                  load another held-out sample
+next                          load the next held-out sample
+quit
+```
+
+At activation position `t`, the readout predicts token `t+1`; the interface
+always prints both tokens to prevent an off-by-one interpretation.
+
+Examples of text export:
+
+```text
+save 200 220                    all layers, top 5, automatic filename
+save 200 220 18                 layer 18 only, top 5
+save 200 220 all 10             all layers, top 10
+save 200 220 18 10 report.txt   layer 18, top 10, explicit filename
+```
+
+`END` is exclusive. Automatic files are written under
+`outputs/jlens-exports/` and include the original token, next predicted token,
+token IDs, mapped J-lens tokens, ranks, and logits.

data/dapo-math-17k/dapo-math-17k.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc9c39c2aa19177abe9464741e121cf4cac90fd25484ef3cdf86535101e3a5b6
+size 10490834

math_jlens.egg-info/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: math-jlens
+Version: 0.1.0
+Requires-Python: >=3.11
+Requires-Dist: torch>=2.6
+Requires-Dist: transformers>=4.53
+Requires-Dist: tqdm>=4.67

math_jlens.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+README.md
+pyproject.toml
+math_jlens/__init__.py
+math_jlens/cli.py
+math_jlens/corpus.py
+math_jlens/fitting.py
+math_jlens/hooks.py
+math_jlens/model.py
+math_jlens.egg-info/PKG-INFO
+math_jlens.egg-info/SOURCES.txt
+math_jlens.egg-info/dependency_links.txt
+math_jlens.egg-info/entry_points.txt
+math_jlens.egg-info/requires.txt
+math_jlens.egg-info/top_level.txt
+tests/test_fitting.py

math_jlens.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

math_jlens.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+fit-jacobians = math_jlens.cli:main

math_jlens.egg-info/requires.txt

@@ -0,0 +1,3 @@
+torch>=2.6
+transformers>=4.53
+tqdm>=4.67

math_jlens.egg-info/top_level.txt

@@ -0,0 +1 @@
+math_jlens

math_jlens/__init__.py

@@ -0,0 +1,6 @@
+"""Math-domain Jacobian lens fitting."""
+
+from .fitting import fit, jacobian_for_tokens
+
+__all__ = ["fit", "jacobian_for_tokens"]
+

math_jlens/cli.py

@@ -0,0 +1,57 @@
+"""Command-line entry point for fitting DAPO Jacobians."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import torch
+
+from .corpus import load_dapo_prompts
+from .fitting import fit
+from .model import load_qwen
+
+
+def build_parser() -> argparse.ArgumentParser:
+    command = argparse.ArgumentParser(description=__doc__)
+    command.add_argument("--model", required=True)
+    command.add_argument("--data", required=True)
+    command.add_argument("--output-dir", default="outputs/jacobians")
+    command.add_argument("--num-prompts", type=int, default=20)
+    command.add_argument("--offset", type=int, default=0)
+    command.add_argument("--seed", type=int, default=17)
+    command.add_argument("--max-seq-len", type=int, default=128)
+    command.add_argument("--skip-first", type=int, default=16)
+    command.add_argument("--dim-batch", type=int, default=8)
+    command.add_argument("--target-layer", type=int)
+    command.add_argument("--checkpoint-every", type=int, default=1)
+    command.add_argument("--device", default="cuda")
+    command.add_argument("--no-resume", action="store_true")
+    return command
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    output = Path(args.output_dir)
+    output.mkdir(parents=True, exist_ok=True)
+    prompts = load_dapo_prompts(args.data, count=args.num_prompts, seed=args.seed, offset=args.offset)
+    model = load_qwen(args.model, device=args.device, dtype=torch.bfloat16)
+    target = model.n_layers - 1 if args.target_layer is None else args.target_layer
+    configuration = vars(args) | {
+        "resolved_target_layer": target, "source_layers": list(range(target)),
+        "model_layers": model.n_layers, "d_model": model.d_model,
+    }
+    (output / "config.json").write_text(json.dumps(configuration, indent=2), encoding="utf-8")
+    fit(
+        model, prompts, output_path=str(output / "lens-bf16.pt"),
+        checkpoint_path=str(output / "fit-checkpoint-fp32.pt"), target_layer=target,
+        max_seq_len=args.max_seq_len, dim_batch=args.dim_batch,
+        skip_first=args.skip_first, checkpoint_every=args.checkpoint_every,
+        resume=not args.no_resume, export_dtype=torch.bfloat16,
+    )
+
+
+if __name__ == "__main__":
+    main()
+

math_jlens/corpus.py

@@ -0,0 +1,34 @@
+"""Deterministic sampling from the DAPO Math JSONL corpus."""
+
+from __future__ import annotations
+
+import json
+import random
+from pathlib import Path
+
+
+def _prompt_text(record: dict) -> str:
+    prompt = record.get("prompt")
+    if isinstance(prompt, str):
+        return prompt
+    if isinstance(prompt, list):
+        parts = [m["content"] for m in prompt if isinstance(m, dict) and isinstance(m.get("content"), str)]
+        if parts:
+            return "\n".join(parts)
+    raise ValueError("record has no supported prompt field")
+
+
+def load_dapo_prompts(path: str, *, count: int, seed: int, offset: int = 0) -> list[str]:
+    records: list[str] = []
+    with Path(path).open(encoding="utf-8") as handle:
+        for line_number, line in enumerate(handle, 1):
+            try:
+                records.append(_prompt_text(json.loads(line)))
+            except (json.JSONDecodeError, ValueError) as exc:
+                raise ValueError(f"invalid DAPO record on line {line_number}: {exc}") from exc
+    random.Random(seed).shuffle(records)
+    selected = records[offset:offset + count]
+    if len(selected) != count:
+        raise ValueError(f"requested {count} prompts at offset {offset}, only {len(selected)} available")
+    return selected
+

math_jlens/explore.py

@@ -0,0 +1,495 @@
+"""Interactive, on-demand J-lens explorer for held-out DAPO Math samples."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import re
+import shlex
+from dataclasses import dataclass
+from pathlib import Path
+
+import torch
+
+from .model import load_qwen
+
+
+def extract_boxed(text: str) -> str | None:
+    """Return the content of the last balanced ``\\boxed{...}``."""
+    marker = r"\boxed{"
+    start = text.rfind(marker)
+    if start < 0:
+        return None
+    content_start = start + len(marker)
+    depth = 1
+    for index in range(content_start, len(text)):
+        if text[index] == "{":
+            depth += 1
+        elif text[index] == "}":
+            depth -= 1
+            if depth == 0:
+                return text[content_start:index]
+    return None
+
+
+def normalize_answer(answer: str | None) -> str | None:
+    if answer is None:
+        return None
+    value = answer.strip().strip("$")
+    value = re.sub(r"\\(?:,|!|;|:|quad|qquad)", "", value)
+    value = value.replace(" ", "").replace(",", "")
+    return value
+
+
+def response_answer(text: str) -> str | None:
+    boxed = extract_boxed(text)
+    if boxed is not None:
+        return boxed
+    matches = re.findall(r"(?im)^\s*Answer\s*:\s*(.+?)\s*$", text)
+    return matches[-1] if matches else None
+
+
+@dataclass(frozen=True)
+class DapoSample:
+    original_index: int
+    messages: list[dict]
+    label: str
+
+
+def load_held_out_sample(path: str, *, seed: int, fit_count: int, sample_index: int) -> DapoSample:
+    records: list[tuple[int, dict]] = []
+    with Path(path).open(encoding="utf-8") as handle:
+        for index, line in enumerate(handle):
+            records.append((index, json.loads(line)))
+    random.Random(seed).shuffle(records)
+    held_out_index = fit_count + sample_index
+    if not 0 <= held_out_index < len(records):
+        raise IndexError(f"held-out sample {sample_index} is outside the dataset")
+    original_index, record = records[held_out_index]
+    prompt = record["prompt"]
+    messages = prompt if isinstance(prompt, list) else [{"role": "user", "content": prompt}]
+    return DapoSample(original_index, messages, str(record["label"]))
+
+
+def find_last_subsequence(sequence: list[int], pattern: list[int]) -> int | None:
+    if not pattern or len(pattern) > len(sequence):
+        return None
+    for start in range(len(sequence) - len(pattern), -1, -1):
+        if sequence[start:start + len(pattern)] == pattern:
+            return start
+    return None
+
+
+class Explorer:
+    def __init__(self, args: argparse.Namespace) -> None:
+        self.args = args
+        self.adapter = load_qwen(args.model, device=args.device, dtype=torch.bfloat16)
+        self.model = self.adapter.model
+        self.tokenizer = self.adapter.tokenizer
+        lens_state = torch.load(args.lens, map_location="cpu", weights_only=True)
+        if lens_state["d_model"] != self.adapter.d_model:
+            raise ValueError(
+                f"lens width {lens_state['d_model']} does not match model width {self.adapter.d_model}"
+            )
+        self.layer_numbers = sorted(lens_state["J"])
+        self.jacobians = {
+            int(layer): matrix.to(args.device)
+            for layer, matrix in lens_state["J"].items()
+        }
+        self.cache_dir = Path(args.cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.activations: dict[int, torch.Tensor] = {}
+        self.sample: DapoSample | None = None
+        self.full_ids: torch.Tensor | None = None
+        self.prompt_length = 0
+        self.response_text = ""
+        self.current_position = 0
+        self.load_sample(args.sample_index)
+
+    def _cache_path(self, sample_index: int) -> Path:
+        return self.cache_dir / f"seed-{self.args.seed}-fit-{self.args.fit_count}-sample-{sample_index}.json"
+
+    def _chat_prompt_ids(self, messages: list[dict]) -> torch.Tensor:
+        ids = self.tokenizer.apply_chat_template(
+            messages,
+            tokenize=True,
+            add_generation_prompt=True,
+            return_tensors="pt",
+        )
+        return ids.to(self.args.device)
+
+    @torch.inference_mode()
+    def _generate(self, sample: DapoSample) -> tuple[list[int], str]:
+        prompt_ids = self._chat_prompt_ids(sample.messages)
+        attention_mask = torch.ones_like(prompt_ids)
+        generated = self.model.generate(
+            input_ids=prompt_ids,
+            attention_mask=attention_mask,
+            do_sample=False,
+            max_new_tokens=self.args.max_new_tokens,
+            use_cache=True,
+            pad_token_id=self.tokenizer.eos_token_id,
+        )
+        response_ids = generated[0, prompt_ids.shape[1]:].tolist()
+        response = self.tokenizer.decode(response_ids, skip_special_tokens=True)
+        return response_ids, response
+
+    def _load_or_generate(self, sample_index: int, sample: DapoSample) -> tuple[list[int], str]:
+        path = self._cache_path(sample_index)
+        if path.exists() and not self.args.regenerate:
+            cached = json.loads(path.read_text(encoding="utf-8"))
+            if cached["original_index"] != sample.original_index:
+                raise ValueError(f"cache identity mismatch at {path}")
+            return cached["response_ids"], cached["response_text"]
+        response_ids, response = self._generate(sample)
+        path.write_text(
+            json.dumps(
+                {
+                    "sample_index": sample_index,
+                    "original_index": sample.original_index,
+                    "label": sample.label,
+                    "response_ids": response_ids,
+                    "response_text": response,
+                    "max_new_tokens": self.args.max_new_tokens,
+                    "do_sample": False,
+                },
+                ensure_ascii=False,
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+        return response_ids, response
+
+    @torch.inference_mode()
+    def _capture(self, full_ids: torch.Tensor) -> None:
+        self.activations.clear()
+        handles = []
+        for layer, block in enumerate(self.adapter.layers):
+            def capture(_module, _inputs, output, layer=layer):
+                tensor = output if torch.is_tensor(output) else output[0]
+                self.activations[layer] = tensor[0].detach()
+            handles.append(block.register_forward_hook(capture))
+        try:
+            self.adapter.forward(full_ids)
+        finally:
+            for handle in handles:
+                handle.remove()
+
+    def load_sample(self, sample_index: int) -> None:
+        sample = load_held_out_sample(
+            self.args.data,
+            seed=self.args.seed,
+            fit_count=self.args.fit_count,
+            sample_index=sample_index,
+        )
+        prompt_ids = self._chat_prompt_ids(sample.messages)
+        response_ids, response = self._load_or_generate(sample_index, sample)
+        full_ids = torch.cat(
+            [prompt_ids, torch.tensor([response_ids], device=self.args.device)], dim=1
+        )
+        self._capture(full_ids)
+        self.args.sample_index = sample_index
+        self.sample = sample
+        self.full_ids = full_ids[0]
+        self.prompt_length = prompt_ids.shape[1]
+        self.response_text = response
+        self.current_position = max(0, self.prompt_length - 1)
+        print(
+            f"Loaded held-out sample {sample_index} (dataset row {sample.original_index})\n"
+            f"prompt_tokens={self.prompt_length} response_tokens={len(response_ids)} "
+            f"total_tokens={len(self.full_ids)}"
+        )
+        self.print_result()
+
+    def problem_text(self) -> str:
+        assert self.sample is not None
+        parts = []
+        for message in self.sample.messages:
+            role = str(message.get("role", "unknown")).upper()
+            content = str(message.get("content", ""))
+            parts.append(f"[{role}]\n{content}")
+        return "\n\n".join(parts)
+
+    def print_problem(self) -> None:
+        print("=== Problem ===")
+        print(self.problem_text())
+
+    def print_output(self) -> None:
+        print("=== Model output ===")
+        print(self.response_text)
+
+    def print_result(self) -> None:
+        assert self.sample is not None
+        predicted = response_answer(self.response_text)
+        normalized_prediction = normalize_answer(predicted)
+        normalized_gold = normalize_answer(self.sample.label)
+        correct = normalized_prediction == normalized_gold
+        print("=== Answers ===")
+        print(f"Extracted:    {predicted!r}")
+        print(f"Ground truth: {self.sample.label!r}")
+        print(f"Correct:      {correct}")
+
+    def print_info(self) -> None:
+        assert self.sample is not None
+        print(
+            f"=== Sample ===\nheld-out index: {self.args.sample_index}\n"
+            f"dataset row:   {self.sample.original_index}"
+        )
+        self.print_problem()
+        self.print_output()
+        self.print_result()
+
+    def _check_position(self, position: int) -> int:
+        assert self.full_ids is not None
+        if position < 0:
+            position += len(self.full_ids)
+        if not 0 <= position < len(self.full_ids) - 1:
+            raise ValueError(f"position must be in [0, {len(self.full_ids) - 2}]")
+        return position
+
+    @torch.inference_mode()
+    def logits(self, layer: int, position: int, *, use_jacobian: bool = True) -> torch.Tensor:
+        residual = self.activations[layer][position]
+        if use_jacobian:
+            residual = residual @ self.jacobians[layer].T
+        residual = self.adapter.decoder.norm(residual)
+        return self.model.lm_head(residual).float()
+
+    def token_label(self, token_id: int) -> str:
+        return repr(self.tokenizer.decode([int(token_id)]))
+
+    def print_tokens(self, start: int | None = None, end: int | None = None) -> None:
+        assert self.full_ids is not None
+        if start is None:
+            start = self.prompt_length
+        if end is None:
+            end = min(len(self.full_ids), start + 80)
+        start = max(0, start)
+        end = min(len(self.full_ids), end)
+        for position in range(start, end):
+            marker = ">" if position == self.current_position else " "
+            region = "R" if position >= self.prompt_length else "P"
+            print(f"{marker} {position:5d} {region} {self.token_label(self.full_ids[position])}")
+
+    def inspect(self, position: int, top_k: int = 5) -> None:
+        position = self._check_position(position)
+        self.current_position = position
+        assert self.full_ids is not None
+        print(
+            f"position={position} current={self.token_label(self.full_ids[position])} "
+            f"predicts={self.token_label(self.full_ids[position + 1])}"
+        )
+        for layer in self.layer_numbers:
+            logits = self.logits(layer, position)
+            values, ids = logits.topk(top_k)
+            decoded = " ".join(
+                f"{self.token_label(token_id)}({value:.2f})"
+                for token_id, value in zip(ids.tolist(), values.tolist(), strict=True)
+            )
+            print(f"L{layer:02d} J: {decoded}")
+
+    def compare(self, position: int, layer: int, top_k: int = 10) -> None:
+        position = self._check_position(position)
+        if layer not in self.layer_numbers:
+            raise ValueError(f"layer must be one of {self.layer_numbers}")
+        for name, enabled in (("J-lens", True), ("logit", False)):
+            values, ids = self.logits(layer, position, use_jacobian=enabled).topk(top_k)
+            decoded = " ".join(
+                f"{self.token_label(token_id)}({value:.2f})"
+                for token_id, value in zip(ids.tolist(), values.tolist(), strict=True)
+            )
+            print(f"{name:7s}: {decoded}")
+
+    @torch.inference_mode()
+    def save_interval(
+        self,
+        start: int,
+        end: int,
+        layer: int | None = None,
+        top_k: int = 5,
+        output_path: str | None = None,
+    ) -> Path:
+        """Export original tokens and their J-lens mappings for ``[start, end)``."""
+        assert self.full_ids is not None and self.sample is not None
+        if start < 0 or end <= start or end > len(self.full_ids) - 1:
+            raise ValueError(
+                f"require 0 <= START < END <= {len(self.full_ids) - 1}; "
+                "END is exclusive"
+            )
+        if top_k < 1:
+            raise ValueError("TOP_K must be positive")
+        layers = self.layer_numbers if layer is None else [layer]
+        if any(item not in self.layer_numbers for item in layers):
+            raise ValueError(f"layer must be one of {self.layer_numbers}, or 'all'")
+
+        layer_label = "all" if layer is None else str(layer)
+        if output_path is None:
+            output = Path(self.args.export_dir) / (
+                f"sample-{self.args.sample_index}-tokens-{start}-{end}-layer-{layer_label}.txt"
+            )
+        else:
+            output = Path(output_path)
+        output.parent.mkdir(parents=True, exist_ok=True)
+
+        lines = [
+            "J-lens token interval export",
+            f"held_out_sample: {self.args.sample_index}",
+            f"dataset_row: {self.sample.original_index}",
+            f"ground_truth: {self.sample.label!r}",
+            f"token_interval: [{start}, {end})",
+            f"layers: {layer_label}",
+            f"top_k: {top_k}",
+            "convention: activation at position t predicts token at t+1",
+            "",
+        ]
+        for position in range(start, end):
+            current_id = int(self.full_ids[position])
+            next_id = int(self.full_ids[position + 1])
+            region = "response" if position >= self.prompt_length else "prompt"
+            lines.extend(
+                [
+                    f"POSITION {position} ({region})",
+                    f"  original: id={current_id} token={self.token_label(current_id)}",
+                    f"  predicts: id={next_id} token={self.token_label(next_id)}",
+                ]
+            )
+            for layer_number in layers:
+                values, ids = self.logits(layer_number, position).topk(top_k)
+                mapped = " | ".join(
+                    f"rank={rank} id={token_id} token={self.token_label(token_id)} logit={value:.4f}"
+                    for rank, (token_id, value) in enumerate(
+                        zip(ids.tolist(), values.tolist(), strict=True), start=1
+                    )
+                )
+                lines.append(f"  L{layer_number:02d}: {mapped}")
+            lines.append("")
+        output.write_text("\n".join(lines), encoding="utf-8")
+        print(f"saved {end - start} positions × {len(layers)} layers to {output}")
+        return output
+
+    def _token_ids(self, text: str) -> list[int]:
+        return self.tokenizer.encode(text, add_special_tokens=False)
+
+    def trace(self, text: str, position: int | None = None) -> None:
+        position = self.current_position if position is None else self._check_position(position)
+        ids = self._token_ids(text)
+        if not ids:
+            raise ValueError("text tokenized to no tokens")
+        print(f"trace text={text!r} ids={ids} at position={position}")
+        for layer in self.layer_numbers:
+            logits = self.logits(layer, position)
+            details = []
+            for token_id in ids:
+                rank = 1 + int((logits > logits[token_id]).sum().item())
+                details.append(f"{self.token_label(token_id)} rank={rank} logit={logits[token_id]:.2f}")
+            print(f"L{layer:02d}: {'; '.join(details)}")
+
+    def answer_trace(self) -> None:
+        assert self.sample is not None and self.full_ids is not None
+        full_list = self.full_ids.tolist()
+        candidates = [self.sample.label, " " + self.sample.label]
+        match = None
+        matched_ids = None
+        for candidate in candidates:
+            ids = self._token_ids(candidate)
+            found = find_last_subsequence(full_list[self.prompt_length:], ids)
+            if found is not None:
+                match = self.prompt_length + found
+                matched_ids = ids
+                break
+        if match is None or matched_ids is None:
+            print(f"Gold answer {self.sample.label!r} was not found as a token sequence in the response")
+            return
+        activation_position = match - 1
+        print(
+            f"gold answer starts at token position {match}; inspecting position "
+            f"{activation_position}, which predicts its first token"
+        )
+        self.trace(self.tokenizer.decode([matched_ids[0]]), activation_position)
+
+    def repl(self) -> None:
+        print("Type 'help' for commands.")
+        while True:
+            try:
+                raw = input("jlens> ").strip()
+                if not raw:
+                    continue
+                parts = shlex.split(raw)
+                command, values = parts[0].lower(), parts[1:]
+                if command in {"quit", "exit", "q"}:
+                    return
+                if command == "help":
+                    print(
+                        "tokens [start] [end]\ninspect POSITION [TOP_K]\n"
+                        "compare POSITION LAYER [TOP_K]\ntrace TEXT [POSITION]\n"
+                        "save START END [LAYER|all] [TOP_K] [FILE]\n"
+                        "problem\noutput\nresult\ninfo\nanswer\n"
+                        "sample INDEX\nnext\nquit"
+                    )
+                elif command == "tokens":
+                    self.print_tokens(*(int(value) for value in values))
+                elif command == "inspect":
+                    self.inspect(int(values[0]), int(values[1]) if len(values) > 1 else 5)
+                elif command == "compare":
+                    self.compare(int(values[0]), int(values[1]), int(values[2]) if len(values) > 2 else 10)
+                elif command == "trace":
+                    self.trace(values[0], int(values[1]) if len(values) > 1 else None)
+                elif command == "save":
+                    if len(values) < 2:
+                        raise ValueError(
+                            "usage: save START END [LAYER|all] [TOP_K] [FILE]"
+                        )
+                    chosen_layer = None
+                    if len(values) > 2 and values[2].lower() != "all":
+                        chosen_layer = int(values[2])
+                    chosen_top_k = int(values[3]) if len(values) > 3 else 5
+                    chosen_file = values[4] if len(values) > 4 else None
+                    self.save_interval(
+                        int(values[0]), int(values[1]), chosen_layer,
+                        chosen_top_k, chosen_file,
+                    )
+                elif command == "answer":
+                    self.answer_trace()
+                elif command == "problem":
+                    self.print_problem()
+                elif command == "output":
+                    self.print_output()
+                elif command == "result":
+                    self.print_result()
+                elif command == "sample":
+                    self.load_sample(int(values[0]))
+                elif command == "next":
+                    self.load_sample(self.args.sample_index + 1)
+                elif command == "info":
+                    self.print_info()
+                else:
+                    print(f"unknown command: {command}")
+            except (IndexError, ValueError) as exc:
+                print(f"error: {exc}")
+            except KeyboardInterrupt:
+                print("\nUse 'quit' to exit.")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--data", required=True)
+    parser.add_argument("--lens", required=True)
+    parser.add_argument("--fit-count", type=int, required=True)
+    parser.add_argument("--seed", type=int, default=17)
+    parser.add_argument("--sample-index", type=int, default=0)
+    parser.add_argument("--device", default="cuda:0")
+    parser.add_argument("--max-new-tokens", type=int, default=2048)
+    parser.add_argument("--cache-dir", default="outputs/explorer-cache")
+    parser.add_argument("--export-dir", default="outputs/jlens-exports")
+    parser.add_argument("--regenerate", action="store_true")
+    return parser
+
+
+def main() -> None:
+    Explorer(build_parser().parse_args()).repl()
+
+
+if __name__ == "__main__":
+    main()

math_jlens/fitting.py

@@ -0,0 +1,160 @@
+"""Averaged input-output Jacobian estimator.
+
+This matches Anthropic's released Jacobian Lens estimator (Apache-2.0): each
+cotangent row is injected at every valid target position, and source-position
+gradients are averaged. ``dim_batch`` rows are computed in parallel.
+"""
+
+from __future__ import annotations
+
+import math
+import os
+from collections.abc import Sequence
+from pathlib import Path
+
+import torch
+from tqdm import tqdm
+
+from .hooks import ActivationRecorder
+
+
+def valid_position_mask(seq_len: int, skip_first: int = 16) -> torch.Tensor:
+    if skip_first < 0:
+        raise ValueError("skip_first must be nonnegative")
+    mask = torch.zeros(seq_len, dtype=torch.bool)
+    mask[skip_first:seq_len - 1] = True
+    if not mask.any():
+        raise ValueError(f"sequence length {seq_len} leaves no positions after skip_first={skip_first}")
+    return mask
+
+
+def jacobian_for_tokens(
+    model,
+    input_ids: torch.Tensor,
+    *,
+    source_layers: Sequence[int],
+    target_layer: int,
+    dim_batch: int = 8,
+    skip_first: int = 16,
+) -> tuple[dict[int, torch.Tensor], int]:
+    """Return per-prompt FP32 CPU Jacobians and valid-position count."""
+    sources = sorted(set(source_layers))
+    if not sources or sources[0] < 0 or sources[-1] >= target_layer:
+        raise ValueError("source layers must be nonempty and below target_layer")
+    if target_layer >= model.n_layers:
+        raise ValueError("target_layer is outside the model")
+    if dim_batch < 1:
+        raise ValueError("dim_batch must be positive")
+
+    width = model.d_model
+    mask = valid_position_mask(input_ids.shape[1], skip_first)
+    jacobians = {layer: torch.zeros(width, width, dtype=torch.float32) for layer in sources}
+    passes = math.ceil(width / dim_batch)
+
+    with ActivationRecorder(model.layers, at=[*sources, target_layer], start_graph_at=sources[0]) as recorder, torch.enable_grad():
+        model.forward(input_ids.expand(dim_batch, -1))
+        target = recorder.activations[target_layer]
+        source_activations = [recorder.activations[layer] for layer in sources]
+        positions = mask.nonzero(as_tuple=True)[0].to(target.device)
+        batch_indices = torch.arange(dim_batch, device=target.device)
+        cotangent = torch.zeros_like(target)
+
+        for pass_index, start in enumerate(range(0, width, dim_batch)):
+            row_count = min(dim_batch, width - start)
+            cotangent.zero_()
+            cotangent[
+                batch_indices[:row_count, None], positions[None, :],
+                start + batch_indices[:row_count, None],
+            ] = 1
+            gradients = torch.autograd.grad(
+                outputs=target, inputs=source_activations, grad_outputs=cotangent,
+                retain_graph=pass_index < passes - 1,
+            )
+            for layer, gradient in zip(sources, gradients, strict=True):
+                local_positions = positions.to(gradient.device)
+                rows = gradient[:row_count, local_positions, :].float().mean(dim=1)
+                jacobians[layer][start:start + row_count] = rows.cpu()
+    return jacobians, int(mask.sum())
+
+
+def _atomic_save(state: dict, path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + f".tmp.{os.getpid()}")
+    torch.save(state, temporary)
+    os.replace(temporary, path)
+
+
+def fit(
+    model,
+    prompts: Sequence[str],
+    *,
+    output_path: str,
+    checkpoint_path: str,
+    source_layers: Sequence[int] | None = None,
+    target_layer: int | None = None,
+    max_seq_len: int = 128,
+    dim_batch: int = 8,
+    skip_first: int = 16,
+    checkpoint_every: int = 1,
+    resume: bool = True,
+    export_dtype: torch.dtype = torch.bfloat16,
+) -> dict:
+    """Fit, checkpoint FP32 sums, and export BF16 averaged matrices."""
+    target = model.n_layers - 1 if target_layer is None else target_layer
+    sources = list(range(target)) if source_layers is None else sorted(set(source_layers))
+    checkpoint = Path(checkpoint_path)
+    metadata_keys = ("source_layers", "target_layer", "d_model", "max_seq_len", "skip_first")
+    expected = (sources, target, model.d_model, max_seq_len, skip_first)
+
+    if resume and checkpoint.exists():
+        state = torch.load(checkpoint, map_location="cpu", weights_only=True)
+        actual = tuple(state[key] for key in metadata_keys)
+        if actual != expected:
+            raise ValueError(f"checkpoint configuration mismatch: {actual} != {expected}")
+    else:
+        state = {
+            "jacobian_sum": {layer: torch.zeros(model.d_model, model.d_model, dtype=torch.float32) for layer in sources},
+            "n_done": 0, "next_index": 0, "source_layers": sources,
+            "target_layer": target, "d_model": model.d_model,
+            "max_seq_len": max_seq_len, "skip_first": skip_first,
+        }
+
+    progress = tqdm(range(state["next_index"], len(prompts)), desc="fitting prompts")
+    for index in progress:
+        input_ids = model.encode(prompts[index], max_seq_len)
+        try:
+            per_prompt, n_valid = jacobian_for_tokens(
+                model, input_ids, source_layers=sources, target_layer=target,
+                dim_batch=dim_batch, skip_first=skip_first,
+            )
+        except ValueError as exc:
+            progress.write(f"skipping prompt {index}: {exc}")
+            state["next_index"] = index + 1
+            continue
+        for layer in sources:
+            state["jacobian_sum"][layer].add_(per_prompt[layer])
+        state["n_done"] += 1
+        state["next_index"] = index + 1
+        progress.set_postfix(valid=n_valid, completed=state["n_done"])
+        if checkpoint_every and state["n_done"] % checkpoint_every == 0:
+            _atomic_save(state, checkpoint)
+
+    _atomic_save(state, checkpoint)
+    if state["n_done"] == 0:
+        raise ValueError("no usable prompts were fitted")
+    exported = {
+        layer: (total / state["n_done"]).to(export_dtype)
+        for layer, total in state["jacobian_sum"].items()
+    }
+    # The transport from the target block output to itself is exactly identity.
+    # Include it so consumers have one matrix for every block-output boundary.
+    exported[target] = torch.eye(model.d_model, dtype=export_dtype)
+    result = {
+        "J": exported,
+        "n_prompts": state["n_done"], "source_layers": sources,
+        "target_layer": target, "d_model": model.d_model,
+        "max_seq_len": max_seq_len, "skip_first": skip_first,
+        "dtype": str(export_dtype), "target_is_identity": True,
+    }
+    _atomic_save(result, Path(output_path))
+    return result

math_jlens/hooks.py

@@ -0,0 +1,40 @@
+"""Forward hooks for residual-stream block outputs.
+
+The estimator follows Anthropic's Apache-2.0 Jacobian Lens reference method:
+https://github.com/anthropics/jacobian-lens
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+
+import torch
+from torch import nn
+
+
+class ActivationRecorder:
+    def __init__(self, blocks: Sequence[nn.Module], at: Iterable[int], *, start_graph_at: int) -> None:
+        self.blocks = blocks
+        self.indices = sorted(set(at) | {start_graph_at})
+        self.start_graph_at = start_graph_at
+        self.activations: dict[int, torch.Tensor] = {}
+        self.handles: list[torch.utils.hooks.RemovableHandle] = []
+
+    def _hook(self, index: int):
+        def record(_module, _inputs, output) -> None:
+            tensor = output if torch.is_tensor(output) else output[0]
+            if index == self.start_graph_at:
+                tensor.requires_grad_(True)
+            self.activations[index] = tensor
+        return record
+
+    def __enter__(self) -> "ActivationRecorder":
+        for index in self.indices:
+            self.handles.append(self.blocks[index].register_forward_hook(self._hook(index)))
+        return self
+
+    def __exit__(self, *_exc) -> None:
+        for handle in self.handles:
+            handle.remove()
+        self.handles.clear()
+

math_jlens/merge.py

@@ -0,0 +1,79 @@
+"""Merge data-parallel FP32 fitting checkpoints into one BF16 lens."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import torch
+
+from .fitting import _atomic_save
+
+
+METADATA_KEYS = (
+    "source_layers",
+    "target_layer",
+    "d_model",
+    "max_seq_len",
+    "skip_first",
+)
+
+
+def merge_checkpoints(paths: list[str], output_path: str) -> dict:
+    if not paths:
+        raise ValueError("at least one checkpoint is required")
+    first = torch.load(paths[0], map_location="cpu", weights_only=True)
+    reference = tuple(first[key] for key in METADATA_KEYS)
+    if first["n_done"] <= 0:
+        raise ValueError("every shard must contain at least one completed prompt")
+    sources = first["source_layers"]
+    target = first["target_layer"]
+    d_model = first["d_model"]
+    totals = {layer: tensor.clone() for layer, tensor in first["jacobian_sum"].items()}
+    shard_counts = [first["n_done"]]
+    del first
+
+    for path in paths[1:]:
+        state = torch.load(path, map_location="cpu", weights_only=True)
+        metadata = tuple(state[key] for key in METADATA_KEYS)
+        if metadata != reference:
+            raise ValueError(f"checkpoint metadata mismatch in {path}: {metadata} != {reference}")
+        if state["n_done"] <= 0:
+            raise ValueError("every shard must contain at least one completed prompt")
+        for layer in sources:
+            totals[layer].add_(state["jacobian_sum"][layer])
+        shard_counts.append(state["n_done"])
+        del state
+
+    total_prompts = sum(shard_counts)
+    means = {layer: (total / total_prompts).to(torch.bfloat16) for layer, total in totals.items()}
+    means[target] = torch.eye(d_model, dtype=torch.bfloat16)
+
+    result = {
+        "J": means,
+        "n_prompts": total_prompts,
+        "source_layers": sources,
+        "target_layer": target,
+        "d_model": d_model,
+        "max_seq_len": reference[3],
+        "skip_first": reference[4],
+        "dtype": str(torch.bfloat16),
+        "target_is_identity": True,
+        "merged_from": [str(Path(path)) for path in paths],
+        "shard_prompt_counts": shard_counts,
+    }
+    _atomic_save(result, Path(output_path))
+    return result
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--checkpoints", nargs="+", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+    result = merge_checkpoints(args.checkpoints, args.output)
+    print(f"merged {len(args.checkpoints)} shards and {result['n_prompts']} prompts into {args.output}")
+
+
+if __name__ == "__main__":
+    main()

math_jlens/model.py

@@ -0,0 +1,47 @@
+"""Minimal Hugging Face adapter for Qwen decoder-only models."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+from torch import nn
+
+
+class QwenLensModel:
+    def __init__(self, model: nn.Module, tokenizer: Any) -> None:
+        self.model = model
+        self.tokenizer = tokenizer
+        self.decoder = model.model
+        self.layers = self.decoder.layers
+        self.n_layers = len(self.layers)
+        self.d_model = model.config.hidden_size
+        model.eval()
+        for parameter in model.parameters():
+            parameter.requires_grad_(False)
+
+    @property
+    def input_device(self) -> torch.device:
+        return self.decoder.embed_tokens.weight.device
+
+    def forward(self, input_ids: torch.Tensor) -> Any:
+        return self.decoder(input_ids=input_ids, use_cache=False)
+
+    def encode(self, text: str, max_length: int) -> torch.Tensor:
+        encoded = self.tokenizer(
+            text, return_tensors="pt", truncation=True, max_length=max_length,
+            add_special_tokens=True,
+        )
+        return encoded.input_ids.to(self.input_device)
+
+
+def load_qwen(model_path: str, *, device: str, dtype: torch.dtype) -> QwenLensModel:
+    """Load a complete local checkpoint without network requests."""
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=True)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_path, local_files_only=True, torch_dtype=dtype
+    ).to(device)
+    return QwenLensModel(model, tokenizer)
+

outputs/explorer-cache/seed-17-fit-20-sample-0.json

@@ -0,0 +1,2058 @@
+{
+  "sample_index": 0,
+  "original_index": 8040,
+  "label": "31",
+  "response_ids": [
+    151667,
+    198,
+    32313,
+    11,
+    773,
+    358,
+    1184,
+    311,
+    11625,
+    419,
+    1849,
+    315,
+    37906,
+    448,
+    2326,
+    7332,
+    856,
+    11,
+    379,
+    11,
+    1147,
+    13,
+    2009,
+    315,
+    1105,
+    525,
+    6785,
+    1931,
+    5109,
+    13,
+    576,
+    37906,
+    525,
+    1447,
+    16,
+    13,
+    400,
+    856,
+    7021,
+    61,
+    17,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    61,
+    17,
+    8,
+    284,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    24437,
+    17,
+    13,
+    400,
+    379,
+    13174,
+    61,
+    17,
+    488,
+    71322,
+    488,
+    856,
+    61,
+    17,
+    8,
+    284,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    24437,
+    18,
+    13,
+    400,
+    1147,
+    2075,
+    61,
+    17,
+    488,
+    30784,
+    488,
+    379,
+    61,
+    17,
+    8,
+    284,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    84500,
+    3036,
+    279,
+    5795,
+    374,
+    311,
+    1477,
+    400,
+    30784,
+    488,
+    379,
+    89,
+    488,
+    71322,
+    400,
+    382,
+    80022,
+    11,
+    1493,
+    37906,
+    1401,
+    5020,
+    54343,
+    11,
+    714,
+    807,
+    7629,
+    944,
+    6896,
+    54343,
+    13,
+    8886,
+    23606,
+    702,
+    264,
+    76002,
+    58441,
+    315,
+    7332,
+    13,
+    10696,
+    1052,
+    594,
+    264,
+    1616,
+    311,
+    32194,
+    429,
+    45120,
+    30,
+    6771,
+    752,
+    1744,
+    382,
+    5338,
+    11,
+    1077,
+    752,
+    5185,
+    429,
+    1817,
+    23606,
+    702,
+    264,
+    79151,
+    4647,
+    304,
+    279,
+    2115,
+    24413,
+    3108,
+    320,
+    43,
+    11961,
+    8,
+    323,
+    13482,
+    3793,
+    389,
+    279,
+    1290,
+    24413,
+    3108,
+    320,
+    49,
+    11961,
+    568,
+    576,
+    79151,
+    3793,
+    525,
+    315,
+    279,
+    1352,
+    400,
+    264,
+    61,
+    17,
+    488,
+    668,
+    488,
+    293,
+    61,
+    17,
+    400,
+    11,
+    892,
+    374,
+    264,
+    4185,
+    7493,
+    304,
+    54343,
+    37906,
+    13,
+    10696,
+    358,
+    646,
+    28143,
+    1493,
+    37906,
+    16925,
+    1939,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    9658,
+    1045,
+    11341,
+    1948,
+    856,
+    11,
+    379,
+    11,
+    1147,
+    30,
+    1752,
+    3110,
+    11,
+    22477,
+    429,
+    856,
+    11,
+    379,
+    11,
+    1147,
+    525,
+    54172,
+    311,
+    1045,
+    18021,
+    13,
+    6771,
+    752,
+    1430,
+    311,
+    9658,
+    429,
+    856,
+    284,
+    595,
+    353,
+    264,
+    11,
+    379,
+    284,
+    595,
+    353,
+    293,
+    11,
+    1147,
+    284,
+    595,
+    353,
+    272,
+    11,
+    1380,
+    264,
+    11,
+    293,
+    11,
+    272,
+    525,
+    18021,
+    323,
+    595,
+    374,
+    264,
+    6785,
+    1931,
+    1372,
+    13,
+    5005,
+    11,
+    31334,
+    10607,
+    1119,
+    279,
+    37906,
+    11,
+    7196,
+    358,
+    646,
+    1477,
+    264,
+    5025,
+    1948,
+    264,
+    11,
+    293,
+    11,
+    272,
+    1939,
+    3983,
+    1573,
+    29002,
+    1119,
+    429,
+    11,
+    1077,
+    752,
+    1779,
+    421,
+    1052,
+    594,
+    264,
+    48968,
+    476,
+    33686,
+    429,
+    646,
+    39721,
+    279,
+    37906,
+    13,
+    6771,
+    752,
+    1401,
+    518,
+    279,
+    37906,
+    1549,
+    1447,
+    24509,
+    367,
+    220,
+    16,
+    25,
+    400,
+    856,
+    7021,
+    61,
+    17,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    61,
+    17,
+    8,
+    284,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    84500,
+    24509,
+    367,
+    220,
+    17,
+    25,
+    400,
+    379,
+    13174,
+    61,
+    17,
+    488,
+    71322,
+    488,
+    856,
+    61,
+    17,
+    8,
+    284,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    84500,
+    24509,
+    367,
+    220,
+    18,
+    25,
+    400,
+    1147,
+    2075,
+    61,
+    17,
+    488,
+    30784,
+    488,
+    379,
+    61,
+    17,
+    8,
+    284,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    84500,
+    22464,
+    518,
+    279,
+    36829,
+    389,
+    279,
+    73832,
+    25,
+    220,
+    18,
+    11,
+    220,
+    16,
+    15,
+    26,
+    220,
+    17,
+    16,
+    11,
+    220,
+    17,
+    19,
+    26,
+    220,
+    22,
+    11,
+    220,
+    17,
+    23,
+    13,
+    13824,
+    11,
+    220,
+    17,
+    16,
+    323,
+    220,
+    17,
+    19,
+    614,
+    264,
+    4185,
+    8168,
+    315,
+    220,
+    18,
+    11,
+    220,
+    22,
+    323,
+    220,
+    17,
+    23,
+    614,
+    264,
+    4185,
+    8168,
+    315,
+    220,
+    22,
+    13,
+    10696,
+    1052,
+    594,
+    264,
+    5383,
+    1588,
+    1939,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    21749,
+    279,
+    37906,
+    553,
+    2494,
+    30,
+    6771,
+    752,
+    1779,
+    279,
+    36829,
+    1549,
+    13,
+    1752,
+    3110,
+    11,
+    23606,
+    220,
+    17,
+    702,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    13,
+    1416,
+    358,
+    8168,
+    220,
+    18,
+    11,
+    432,
+    594,
+    220,
+    18,
+    6599,
+    22,
+    89,
+    488,
+    220,
+    23,
+    87,
+    568,
+    34239,
+    11,
+    23606,
+    220,
+    18,
+    702,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    11,
+    892,
+    374,
+    220,
+    22,
+    6599,
+    87,
+    488,
+    220,
+    19,
+    88,
+    568,
+    84670,
+    220,
+    16,
+    702,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    13,
+    2806,
+    2704,
+    421,
+    429,
+    8609,
+    3602,
+    382,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    2908,
+    41238,
+    1948,
+    37906,
+    30,
+    6771,
+    752,
+    1744,
+    13,
+    6771,
+    752,
+    78064,
+    279,
+    37906,
+    438,
+    33122,
+    16,
+    11,
+    33122,
+    17,
+    11,
+    33122,
+    18,
+    382,
+    10061,
+    752,
+    1430,
+    311,
+    3158,
+    1817,
+    23606,
+    304,
+    3793,
+    315,
+    7332,
+    13,
+    6771,
+    752,
+    1744,
+    911,
+    3204,
+    93650,
+    13,
+    1752,
+    3110,
+    11,
+    7196,
+    358,
+    646,
+    3158,
+    856,
+    11,
+    379,
+    11,
+    1147,
+    304,
+    3793,
+    315,
+    1817,
+    1008,
+    1939,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    2908,
+    279,
+    41238,
+    315,
+    279,
+    37906,
+    13,
+    1752,
+    3110,
+    11,
+    421,
+    358,
+    1896,
+    33122,
+    16,
+    17779,
+    553,
+    33122,
+    17,
+    476,
+    2494,
+    1075,
+    429,
+    13,
+    1988,
+    2474,
+    807,
+    525,
+    2155,
+    37906,
+    11,
+    7196,
+    537,
+    30339,
+    382,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    9658,
+    429,
+    7332,
+    525,
+    304,
+    1045,
+    21117,
+    13,
+    6771,
+    752,
+    1430,
+    311,
+    9658,
+    429,
+    856,
+    11,
+    379,
+    11,
+    1147,
+    525,
+    54172,
+    311,
+    1045,
+    5109,
+    13,
+    6771,
+    752,
+    1744,
+    911,
+    279,
+    36829,
+    304,
+    279,
+    73832,
+    13,
+    1752,
+    3110,
+    11,
+    304,
+    23606,
+    220,
+    16,
+    11,
+    279,
+    73832,
+    374,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    13,
+    758,
+    23606,
+    220,
+    17,
+    11,
+    432,
+    594,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    13,
+    758,
+    23606,
+    220,
+    18,
+    11,
+    432,
+    594,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    13,
+    6771,
+    752,
+    5185,
+    429,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    646,
+    387,
+    5326,
+    438,
+    220,
+    18,
+    6599,
+    22,
+    89,
+    488,
+    220,
+    23,
+    87,
+    701,
+    323,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    374,
+    220,
+    22,
+    6599,
+    87,
+    488,
+    220,
+    19,
+    88,
+    568,
+    34239,
+    11,
+    23606,
+    220,
+    16,
+    374,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    13,
+    10696,
+    1052,
+    594,
+    264,
+    1616,
+    311,
+    28143,
+    1493,
+    1939,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    1744,
+    315,
+    279,
+    37906,
+    438,
+    13482,
+    37906,
+    304,
+    3793,
+    315,
+    7332,
+    30,
+    13824,
+    11,
+    902,
+    11,
+    1576,
+    279,
+    444,
+    11961,
+    374,
+    79151,
+    13,
+    2055,
+    807,
+    525,
+    73998,
+    37906,
+    382,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    2908,
+    31334,
+    10607,
+    7332,
+    13,
+    6771,
+    752,
+    1744,
+    13,
+    6771,
+    752,
+    78064,
+    328,
+    284,
+    30784,
+    488,
+    379,
+    89,
+    488,
+    71322,
+    11,
+    892,
+    374,
+    279,
+    12194,
+    582,
+    1184,
+    311,
+    1477,
+    13,
+    10696,
+    358,
+    646,
+    1477,
+    23393,
+    15860,
+    328,
+    1939,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    2908,
+    279,
+    37906,
+    438,
+    11017,
+    13,
+    6771,
+    752,
+    5185,
+    429,
+    279,
+    79151,
+    3793,
+    389,
+    279,
+    444,
+    11961,
+    525,
+    4428,
+    311,
+    279,
+    7493,
+    369,
+    279,
+    9334,
+    315,
+    279,
+    2629,
+    476,
+    2494,
+    770,
+    13,
+    13824,
+    11,
+    369,
+    3110,
+    11,
+    400,
+    379,
+    61,
+    17,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    61,
+    17,
+    284,
+    1124,
+    37018,
+    90,
+    18,
+    15170,
+    19,
+    25547,
+    88,
+    488,
+    1147,
+    29776,
+    17,
+    488,
+    1124,
+    37018,
+    90,
+    16,
+    15170,
+    19,
+    25547,
+    88,
+    481,
+    1147,
+    29776,
+    17,
+    400,
+    11,
+    714,
+    537,
+    2704,
+    421,
+    429,
+    8609,
+    1588,
+    382,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    1744,
+    315,
+    279,
+    37906,
+    438,
+    1447,
+    2461,
+    23606,
+    220,
+    16,
+    25,
+    400,
+    856,
+    7021,
+    61,
+    17,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    61,
+    17,
+    8,
+    481,
+    220,
+    18,
+    88,
+    481,
+    220,
+    16,
+    15,
+    89,
+    284,
+    220,
+    15,
+    84500,
+    67691,
+    369,
+    3800,
+    13,
+    10696,
+    358,
+    646,
+    1744,
+    315,
+    419,
+    438,
+    264,
+    1849,
+    1380,
+    1817,
+    23606,
+    374,
+    264,
+    10601,
+    315,
+    7332,
+    13,
+    1988,
+    21828,
+    1741,
+    264,
+    1849,
+    4977,
+    16965,
+    382,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    9658,
+    429,
+    7332,
+    525,
+    25780,
+    30,
+    6771,
+    752,
+    1779,
+    421,
+    429,
+    594,
+    3204,
+    13,
+    6771,
+    752,
+    1430,
+    311,
+    1490,
+    421,
+    1052,
+    525,
+    7546,
+    9904,
+    13,
+    6771,
+    752,
+    22477,
+    429,
+    856,
+    11,
+    379,
+    11,
+    1147,
+    525,
+    25780,
+    13,
+    6771,
+    752,
+    1779,
+    23606,
+    220,
+    18,
+    25,
+    1147,
+    2075,
+    29456,
+    488,
+    30784,
+    488,
+    379,
+    29456,
+    8,
+    284,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    13,
+    6771,
+    752,
+    5185,
+    429,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    284,
+    220,
+    22,
+    2075,
+    488,
+    220,
+    19,
+    88,
+    568,
+    2055,
+    1147,
+    64828,
+    220,
+    22,
+    2075,
+    488,
+    220,
+    19,
+    88,
+    568,
+    34239,
+    11,
+    23606,
+    220,
+    17,
+    25,
+    379,
+    13174,
+    29456,
+    488,
+    71322,
+    488,
+    856,
+    29456,
+    8,
+    284,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    13,
+    6771,
+    752,
+    5185,
+    429,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    284,
+    220,
+    18,
+    7,
+    22,
+    89,
+    488,
+    220,
+    23,
+    87,
+    568,
+    2055,
+    379,
+    64828,
+    220,
+    18,
+    7,
+    22,
+    89,
+    488,
+    220,
+    23,
+    87,
+    568,
+    34239,
+    11,
+    23606,
+    220,
+    16,
+    25,
+    856,
+    7021,
+    29456,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    29456,
+    8,
+    284,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    13,
+    2055,
+    856,
+    64828,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    382,
+    3983,
+    2474,
+    856,
+    11,
+    379,
+    11,
+    1147,
+    525,
+    6785,
+    25780,
+    11,
+    7196,
+    582,
+    646,
+    1477,
+    1045,
+    4300,
+    13,
+    4354,
+    11,
+    2474,
+    279,
+    3491,
+    2727,
+    807,
+    525,
+    6785,
+    1931,
+    5109,
+    11,
+    537,
+    14312,
+    25780,
+    11,
+    773,
+    7196,
+    419,
+    5486,
+    374,
+    2238,
+    56996,
+    13,
+    1988,
+    7196,
+    1052,
+    594,
+    264,
+    6291,
+    448,
+    7546,
+    2750,
+    30,
+    6771,
+    752,
+    1430,
+    311,
+    1490,
+    382,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    1401,
+    369,
+    41238,
+    1948,
+    7332,
+    13,
+    6771,
+    752,
+    22477,
+    429,
+    1052,
+    374,
+    264,
+    11341,
+    1948,
+    856,
+    11,
+    379,
+    11,
+    1147,
+    13,
+    6771,
+    752,
+    9658,
+    429,
+    856,
+    284,
+    264,
+    11,
+    379,
+    284,
+    293,
+    11,
+    1147,
+    284,
+    272,
+    11,
+    323,
+    1430,
+    311,
+    1477,
+    41238,
+    13,
+    6771,
+    752,
+    1744,
+    382,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    2908,
+    49702,
+    37906,
+    13,
+    1752,
+    3110,
+    11,
+    1896,
+    23606,
+    220,
+    16,
+    323,
+    23606,
+    220,
+    18,
+    13,
+    6771,
+    752,
+    1490,
+    1447,
+    24509,
+    367,
+    220,
+    16,
+    25,
+    856,
+    7021,
+    29456,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    29456,
+    8,
+    284,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    271,
+    24509,
+    367,
+    220,
+    18,
+    25,
+    1147,
+    2075,
+    29456,
+    488,
+    30784,
+    488,
+    379,
+    29456,
+    8,
+    284,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    271,
+    2679,
+    358,
+    21749,
+    23606,
+    220,
+    16,
+    553,
+    23606,
+    220,
+    18,
+    11,
+    358,
+    633,
+    1447,
+    58,
+    856,
+    7021,
+    29456,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    29456,
+    8,
+    2279,
+    608,
+    508,
+    1147,
+    2075,
+    29456,
+    488,
+    30784,
+    488,
+    379,
+    29456,
+    8,
+    2279,
+    284,
+    320,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    24254,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    692,
+    50,
+    70206,
+    63533,
+    323,
+    46912,
+    1447,
+    5415,
+    3108,
+    25,
+    508,
+    87,
+    31082,
+    60,
+    353,
+    508,
+    320,
+    88,
+    29456,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    29456,
+    8,
+    608,
+    320,
+    87,
+    29456,
+    488,
+    30784,
+    488,
+    379,
+    29456,
+    8,
+    10444,
+    5979,
+    3108,
+    25,
+    508,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    9769,
+    58,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    60,
+    284,
+    508,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    9769,
+    58,
+    22,
+    2075,
+    488,
+    220,
+    19,
+    88,
+    27771,
+    80022,
+    11,
+    537,
+    2704,
+    421,
+    10950,
+    3602,
+    13,
+    10696,
+    358,
+    646,
+    9658,
+    1045,
+    11341,
+    1948,
+    7332,
+    13,
+    6771,
+    752,
+    22477,
+    429,
+    856,
+    11,
+    379,
+    11,
+    1147,
+    525,
+    304,
+    1045,
+    21117,
+    13,
+    6771,
+    752,
+    1744,
+    911,
+    279,
+    36829,
+    304,
+    279,
+    73832,
+    13,
+    1752,
+    3110,
+    11,
+    23606,
+    220,
+    16,
+    702,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    11,
+    23606,
+    220,
+    17,
+    702,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    11,
+    23606,
+    220,
+    18,
+    702,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    13,
+    6771,
+    752,
+    5185,
+    429,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    284,
+    220,
+    18,
+    6599,
+    22,
+    89,
+    488,
+    220,
+    23,
+    87,
+    701,
+    323,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    284,
+    220,
+    22,
+    6599,
+    87,
+    488,
+    220,
+    19,
+    88,
+    568,
+    7281,
+    11,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    374,
+    438,
+    374,
+    382,
+    92014,
+    11,
+    7196,
+    358,
+    646,
+    1744,
+    315,
+    279,
+    37906,
+    438,
+    11017,
+    1447,
+    10061,
+    752,
+    78064,
+    1447,
+    10061,
+    752,
+    1744,
+    315,
+    23606,
+    220,
+    16,
+    25,
+    856,
+    7021,
+    29456,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    29456,
+    8,
+    284,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    271,
+    24509,
+    367,
+    220,
+    17,
+    25,
+    379,
+    13174,
+    29456,
+    488,
+    71322,
+    488,
+    856,
+    29456,
+    8,
+    284,
+    220,
+    17,
+    16,
+    89,
+    488,
+    220,
+    17,
+    19,
+    87,
+    271,
+    24509,
+    367,
+    220,
+    18,
+    25,
+    1147,
+    2075,
+    29456,
+    488,
+    30784,
+    488,
+    379,
+    29456,
+    8,
+    284,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    271,
+    10061,
+    752,
+    5185,
+    429,
+    279,
+    36829,
+    389,
+    279,
+    73832,
+    315,
+    23606,
+    220,
+    16,
+    525,
+    220,
+    18,
+    323,
+    220,
+    16,
+    15,
+    11,
+    23606,
+    220,
+    17,
+    525,
+    220,
+    17,
+    16,
+    323,
+    220,
+    17,
+    19,
+    11,
+    23606,
+    220,
+    18,
+    525,
+    220,
+    22,
+    323,
+    220,
+    17,
+    23,
+    13,
+    6771,
+    752,
+    1779,
+    421,
+    1052,
+    594,
+    264,
+    5383,
+    1588,
+    13,
+    1752,
+    3110,
+    11,
+    220,
+    18,
+    11,
+    220,
+    16,
+    15,
+    26,
+    220,
+    17,
+    16,
+    11,
+    220,
+    17,
+    19,
+    26,
+    220,
+    22,
+    11,
+    220,
+    17,
+    23,
+    13,
+    6771,
+    752,
+    1490,
+    1447,
+    2679,
+    358,
+    1401,
+    518,
+    23606,
+    220,
+    16,
+    323,
+    23606,
+    220,
+    18,
+    25,
+    23606,
+    220,
+    16,
+    702,
+    220,
+    18,
+    88,
+    488,
+    220,
+    16,
+    15,
+    89,
+    11,
+    23606,
+    220,
+    18,
+    702,
+    220,
+    22,
+    87,
+    488,
+    220,
+    17,
+    23,
+    88,
+    13,
+    1416,
+    358,
+    30270,
+    23606,
+    220,
+    16,
+    553,
+    2494,
+    30,
+    6771,
+    752,
+    1490,
+    1447,
+    2679,
+    358,
+    30270,
+    23606,
+    220,
+    16,
+    553,
+    220,
+    22,
+    11,
+    358,
+    633,
+    220,
+    22,
+    87,
+    7021,
+    29456,
+    488,
+    379,
+    89,
+    488,
+    1147,
+    29456,
+    8,
+    284,
+    220,
+    17,
+    16,
+    88,
+    488,
+    220,
+    22,
+    15,
+    89,
+    13,
+    23768,
+    448,
+    23606,
+    220,
+    18,
+    25,
+    1147,
+    2075,
+    29456,
+    488,
+    30784,
+    488,
+    379,
+    29456,
+    8,
+    284,
+    220,
+    22,
+    87,
+    488
+  ],
+  "response_text": "<think>\nOkay, so I need to solve this system of equations with three variables x, y, z. All of them are positive real numbers. The equations are:\n\n1. $ x(y^2 + yz + z^2) = 3y + 10z $\n2. $ y(z^2 + zx + x^2) = 21z + 24x $\n3. $ z(x^2 + xy + y^2) = 7x + 28y $\n\nAnd the goal is to find $ xy + yz + zx $.\n\nHmm, these equations look pretty symmetric, but they aren't exactly symmetric. Each equation has a cyclic permutation of variables. Maybe there's a way to exploit that symmetry? Let me think.\n\nFirst, let me note that each equation has a quadratic term in the left-hand side (LHS) and linear terms on the right-hand side (RHS). The quadratic terms are of the form $ a^2 + ab + b^2 $, which is a common expression in symmetric equations. Maybe I can relate these equations somehow?\n\nAlternatively, maybe I can assume some ratio between x, y, z? For example, suppose that x, y, z are proportional to some constants. Let me try to assume that x = k * a, y = k * b, z = k * c, where a, b, c are constants and k is a positive real number. Then, substituting into the equations, maybe I can find a relationship between a, b, c?\n\nBut before jumping into that, let me check if there's a substitution or manipulation that can simplify the equations. Let me look at the equations again:\n\nEquation 1: $ x(y^2 + yz + z^2) = 3y + 10z $\n\nEquation 2: $ y(z^2 + zx + x^2) = 21z + 24x $\n\nEquation 3: $ z(x^2 + xy + y^2) = 7x + 28y $\n\nLooking at the coefficients on the RHS: 3, 10; 21, 24; 7, 28. Wait, 21 and 24 have a common factor of 3, 7 and 28 have a common factor of 7. Maybe there's a pattern here?\n\nAlternatively, maybe I can divide the equations by something? Let me check the coefficients again. For example, equation 2 has 21z + 24x. If I factor 3, it's 3*(7z + 8x). Similarly, equation 3 has 7x + 28y, which is 7*(x + 4y). Equation 1 has 3y + 10z. Not sure if that helps yet.\n\nAlternatively, maybe I can consider ratios between equations? Let me think. Let me denote the equations as Eq1, Eq2, Eq3.\n\nLet me try to express each equation in terms of variables. Let me think about possible substitutions. For example, maybe I can express x, y, z in terms of each other?\n\nAlternatively, maybe I can consider the ratios of the equations. For example, if I take Eq1 divided by Eq2 or something like that. But since they are different equations, maybe not straightforward.\n\nAlternatively, maybe I can assume that variables are in some proportion. Let me try to assume that x, y, z are proportional to some numbers. Let me think about the coefficients in the RHS. For example, in equation 1, the RHS is 3y + 10z. In equation 2, it's 21z + 24x. In equation 3, it's 7x + 28y. Let me note that 21z + 24x can be written as 3*(7z + 8x), and 7x + 28y is 7*(x + 4y). Similarly, equation 1 is 3y + 10z. Maybe there's a way to relate these?\n\nAlternatively, maybe I can think of the equations as linear equations in terms of variables? Wait, no, because the LHS is quadratic. So they are nonlinear equations.\n\nAlternatively, maybe I can consider substituting variables. Let me think. Let me denote S = xy + yz + zx, which is the quantity we need to find. Maybe I can find expressions involving S?\n\nAlternatively, maybe I can consider the equations as follows. Let me note that the quadratic terms on the LHS are similar to the expression for the square of the sum or something else. Wait, for example, $ y^2 + yz + z^2 = \\frac{3}{4}(y + z)^2 + \\frac{1}{4}(y - z)^2 $, but not sure if that helps here.\n\nAlternatively, maybe I can think of the equations as:\n\nFor equation 1: $ x(y^2 + yz + z^2) - 3y - 10z = 0 $\n\nSimilarly for others. Maybe I can think of this as a system where each equation is a combination of variables. But solving such a system seems complicated.\n\nAlternatively, maybe I can assume that variables are integers? Let me check if that's possible. Let me try to see if there are integer solutions. Let me suppose that x, y, z are integers. Let me check equation 3: z(x² + xy + y²) = 7x + 28y. Let me note that 7x + 28y = 7(x + 4y). So z divides 7(x + 4y). Similarly, equation 2: y(z² + zx + x²) = 21z + 24x. Let me note that 21z + 24x = 3(7z + 8x). So y divides 3(7z + 8x). Similarly, equation 1: x(y² + yz + z²) = 3y + 10z. So x divides 3y + 10z.\n\nBut since x, y, z are positive integers, maybe we can find some relations. However, since the problem says they are positive real numbers, not necessarily integers, so maybe this approach is too restrictive. But maybe there's a solution with integer values? Let me try to see.\n\nAlternatively, maybe I can look for ratios between variables. Let me suppose that there is a ratio between x, y, z. Let me assume that x = a, y = b, z = c, and try to find ratios. Let me think.\n\nAlternatively, maybe I can consider dividing equations. For example, take equation 1 and equation 3. Let me see:\n\nEquation 1: x(y² + yz + z²) = 3y + 10z\n\nEquation 3: z(x² + xy + y²) = 7x + 28y\n\nIf I divide equation 1 by equation 3, I get:\n\n[ x(y² + yz + z²) ] / [ z(x² + xy + y²) ] = (3y + 10z)/(7x + 28y)\n\nSimplify numerator and denominator:\n\nLeft side: [x/z] * [ (y² + yz + z²) / (x² + xy + y²) ]\n\nRight side: [3y + 10z]/[7x + 28y] = [3y + 10z]/[7(x + 4y)]\n\nHmm, not sure if helpful yet. Maybe I can assume some ratio between variables. Let me suppose that x, y, z are in some proportion. Let me think about the coefficients in the RHS. For example, equation 1 has 3y + 10z, equation 2 has 21z + 24x, equation 3 has 7x + 28y. Let me note that 21z + 24x = 3*(7z + 8x), and 7x + 28y = 7*(x + 4y). Also, 3y + 10z is as is.\n\nAlternatively, maybe I can think of the equations as follows:\n\nLet me denote:\n\nLet me think of equation 1: x(y² + yz + z²) = 3y + 10z\n\nEquation 2: y(z² + zx + x²) = 21z + 24x\n\nEquation 3: z(x² + xy + y²) = 7x + 28y\n\nLet me note that the coefficients on the RHS of equation 1 are 3 and 10, equation 2 are 21 and 24, equation 3 are 7 and 28. Let me check if there's a pattern here. For example, 3, 10; 21, 24; 7, 28. Let me see:\n\nIf I look at equation 1 and equation 3: equation 1 has 3y + 10z, equation 3 has 7x + 28y. If I multiply equation 1 by something? Let me see:\n\nIf I multiply equation 1 by 7, I get 7x(y² + yz + z²) = 21y + 70z. Compare with equation 3: z(x² + xy + y²) = 7x +",
+  "max_new_tokens": 2048,
+  "do_sample": false
+}

outputs/multi-gpu-1000/lens-bf16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad3376c6e7b25412b4cd00a0615e618ff8ea025418792f7c4a8c9dbc375a7298
+size 471870034

outputs/multi-gpu-1000/shard-0/config.json

@@ -0,0 +1,55 @@
+{
+  "model": "../Lightning-OPD/checkpoints/qwen3-4b-base-sft-qwen3-8b",
+  "data": "data/dapo-math-17k/dapo-math-17k.jsonl",
+  "output_dir": "outputs/multi-gpu-1000/shard-0",
+  "num_prompts": 500,
+  "offset": 0,
+  "seed": 17,
+  "max_seq_len": 128,
+  "skip_first": 16,
+  "dim_batch": 64,
+  "target_layer": null,
+  "checkpoint_every": 10,
+  "device": "cuda:0",
+  "no_resume": false,
+  "resolved_target_layer": 35,
+  "source_layers": [
+    0,
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27,
+    28,
+    29,
+    30,
+    31,
+    32,
+    33,
+    34
+  ],
+  "model_layers": 36,
+  "d_model": 2560
+}

outputs/multi-gpu-1000/shard-0/fit-checkpoint-fp32.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3804bdecda73099d8bff5816bac3874ed0137c46c332c4eeeb32a399f578a14
+size 917514770

outputs/multi-gpu-1000/shard-0/lens-bf16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f910d346ad8b35f3e948374dd4b1fd98fe3199ef636f5bb1eb5ccfaacf7496c3
+size 471869842