Upload context-a, context-multi-bios, context-multi-bios-20k, context-multi-bios-20k-s1, dense-a, dense-a-k100, dense-a-to-b, latent-a, latent-a-all, latent-multi, latent-multi-nohop, lookup-a
Browse files- README.md +37 -0
- context-a/checkpoints/final/model.safetensors +3 -0
- context-a/config.json +40 -0
- context-multi-bios-20k-s1/checkpoints/final/model.safetensors +3 -0
- context-multi-bios-20k-s1/config.json +83 -0
- context-multi-bios-20k/checkpoints/final/model.safetensors +3 -0
- context-multi-bios-20k/config.json +83 -0
- context-multi-bios/checkpoints/final/model.safetensors +3 -0
- context-multi-bios/config.json +83 -0
- dense-a-k100/checkpoints/final/model.safetensors +3 -0
- dense-a-k100/config.json +44 -0
- dense-a-to-b/checkpoints/final/model.safetensors +3 -0
- dense-a-to-b/config.json +40 -0
- dense-a/checkpoints/final/model.safetensors +3 -0
- dense-a/config.json +44 -0
- latent-a-all/checkpoints/final/model.safetensors +3 -0
- latent-a-all/config.json +55 -0
- latent-a/checkpoints/final/model.safetensors +3 -0
- latent-a/config.json +54 -0
- latent-multi-nohop/checkpoints/latest/model.safetensors +3 -0
- latent-multi-nohop/config.json +55 -0
- latent-multi/checkpoints/final/model.safetensors +3 -0
- latent-multi/config.json +55 -0
- lookup-a/checkpoints/final/model.safetensors +3 -0
- lookup-a/config.json +44 -0
README.md
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
library_name: mlx
|
| 4 |
+
tags: [mlx, knowledge-editing, retrieval, synthetic-data]
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
# Hemispheres: step-1 checkpoints
|
| 8 |
+
|
| 9 |
+
Weights for the step-1 synthetic-world runs of Hemispheres, a language model that keeps its knowledge in a separate, editable store and reads it inside the forward pass. Each run directory holds `config.json` (every training flag, seed and model shape) and MLX safetensors checkpoints.
|
| 10 |
+
|
| 11 |
+
The run records (metrics, per-question evaluation outputs, data fingerprints), the comparison report and the code to load and evaluate these weights are in the project repository, under `results/step1/`.
|
| 12 |
+
|
| 13 |
+
| run | arm | params | checkpoints | sha256 |
|
| 14 |
+
|---|---|---|---|---|
|
| 15 |
+
| `context-a` | context | 25.5M | `final`, `latest` | `ae48aaa6bc50` |
|
| 16 |
+
| `context-multi-bios` | context | 25.5M | `final`, `latest` | `8d97615b89a9` |
|
| 17 |
+
| `context-multi-bios-20k` | context | 25.5M | `final`, `latest` | `11424cc903c9` |
|
| 18 |
+
| `context-multi-bios-20k-s1` | context | 25.5M | `final`, `latest` | `4e9410c97d16` |
|
| 19 |
+
| `dense-a` | dense | 25.5M | `final`, `latest` | `81e8eb4ac912` |
|
| 20 |
+
| `dense-a-k100` | dense | 25.5M | `final`, `latest` | `c2535e281e08` |
|
| 21 |
+
| `dense-a-to-b` | dense | 25.5M | `final`, `latest` | `17c630e69033` |
|
| 22 |
+
| `latent-a` | latent | 29.2M | `final`, `latest` | `f4f3b1fbe96a` |
|
| 23 |
+
| `latent-a-all` | latent | 29.2M | `final`, `latest` | `57dc7d12714c` |
|
| 24 |
+
| `latent-multi` | latent | 29.2M | `final`, `latest` | `1b2490e51d93` |
|
| 25 |
+
| `latent-multi-nohop` | latent | 29.2M | `latest` | `dc2494c38336` |
|
| 26 |
+
| `lookup-a` | lookup | 25.5M | `final`, `latest` | `e06385a28712` |
|
| 27 |
+
|
| 28 |
+
```sh
|
| 29 |
+
pip install -e '.[hub]'
|
| 30 |
+
python -m hemispheres.records fetch latent-multi # into runs/latent-multi/, sha256-checked
|
| 31 |
+
python -m hemispheres.synth.build --name world-b --index 1 --edits 1,100,1000
|
| 32 |
+
python -m hemispheres.evaluate --run runs/latent-multi --data data/world-b --sets all --n 300
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
Models are 25.5M (dense, lookup, context) or 29.2M (latent) parameters, fp32, trained from scratch on generated worlds. They are research artifacts for the synthetic task, not general-purpose language models.
|
| 36 |
+
|
| 37 |
+
Released under the MIT license (see `LICENSE`).
|
context-a/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ae48aaa6bc50869386a3a0892576d33c5a69ca5fda2dc4c396a36d525ee18d20
|
| 3 |
+
size 101882492
|
context-a/config.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "context",
|
| 3 |
+
"data": "data/world-a",
|
| 4 |
+
"out": "runs/context-a",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"qa",
|
| 9 |
+
1.0
|
| 10 |
+
]
|
| 11 |
+
],
|
| 12 |
+
"init": null,
|
| 13 |
+
"init_checkpoint": "final",
|
| 14 |
+
"size": "small",
|
| 15 |
+
"dtype": "float32",
|
| 16 |
+
"steps": 10000,
|
| 17 |
+
"batch_size": 64,
|
| 18 |
+
"seq_len": 256,
|
| 19 |
+
"lr": 0.001,
|
| 20 |
+
"min_lr_frac": 0.1,
|
| 21 |
+
"warmup": 500,
|
| 22 |
+
"weight_decay": 0.1,
|
| 23 |
+
"grad_clip": 1.0,
|
| 24 |
+
"log_every": 100,
|
| 25 |
+
"eval_every": 2000,
|
| 26 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 27 |
+
"eval_n": 200,
|
| 28 |
+
"save_every": 2000,
|
| 29 |
+
"seed": 0,
|
| 30 |
+
"model": {
|
| 31 |
+
"vocab_size": 576,
|
| 32 |
+
"n_layer": 8,
|
| 33 |
+
"n_head": 8,
|
| 34 |
+
"d_model": 512,
|
| 35 |
+
"mlp_ratio": 4,
|
| 36 |
+
"rope_base": 10000.0
|
| 37 |
+
},
|
| 38 |
+
"params": 25469440,
|
| 39 |
+
"vocab": 544
|
| 40 |
+
}
|
context-multi-bios-20k-s1/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4e9410c97d16943e82ff7794ce5c79245c96202b872aac50a872c5ec8bb77266
|
| 3 |
+
size 101882492
|
context-multi-bios-20k-s1/config.json
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "context",
|
| 3 |
+
"data": "data/world-a,data/pool/w3-s0,data/pool/w3-s1,data/pool/w3-s2,data/pool/w4-s0,data/pool/w4-s1,data/pool/w4-s2,data/pool/w5-s0,data/pool/w5-s1,data/pool/w5-s2,data/pool/w6-s0,data/pool/w6-s1,data/pool/w6-s2,data/pool/w7-s0,data/pool/w7-s1,data/pool/w7-s2",
|
| 4 |
+
"out": "runs/context-multi-bios-20k-s1",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"qa",
|
| 13 |
+
0.5
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": null,
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 20000,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.001,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 500,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 2000,
|
| 30 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 31 |
+
"eval_n": 200,
|
| 32 |
+
"save_every": 2000,
|
| 33 |
+
"seed": 1,
|
| 34 |
+
"n_reads": 3,
|
| 35 |
+
"top_k": 4,
|
| 36 |
+
"hop_weight": 0.5,
|
| 37 |
+
"hop_until": 0,
|
| 38 |
+
"max_supervision": 1024,
|
| 39 |
+
"supervise": "all",
|
| 40 |
+
"model_type": "gpt",
|
| 41 |
+
"model": {
|
| 42 |
+
"vocab_size": 576,
|
| 43 |
+
"n_layer": 8,
|
| 44 |
+
"n_head": 8,
|
| 45 |
+
"d_model": 512,
|
| 46 |
+
"mlp_ratio": 4,
|
| 47 |
+
"rope_base": 10000.0
|
| 48 |
+
},
|
| 49 |
+
"params": 25469440,
|
| 50 |
+
"vocab": 544,
|
| 51 |
+
"provenance": {
|
| 52 |
+
"time": "2026-09-25T20:10:18+00:00",
|
| 53 |
+
"code": {
|
| 54 |
+
"commit": "4fc4e676ef82fd2bb4ea3bddc5f4471bc12f3e93",
|
| 55 |
+
"dirty": false
|
| 56 |
+
},
|
| 57 |
+
"env": {
|
| 58 |
+
"python": "3.12.11",
|
| 59 |
+
"platform": "macOS-26.6.2-arm64-arm-64bit",
|
| 60 |
+
"machine": "arm64",
|
| 61 |
+
"mlx": "0.32.2",
|
| 62 |
+
"device": "Device(gpu, 0)"
|
| 63 |
+
},
|
| 64 |
+
"data": {
|
| 65 |
+
"data/world-a": "befab6f0a218e36f48e7ae53539eef4e3fe9fab49eaae9a2d603ee01b86df106",
|
| 66 |
+
"data/pool/w3-s0": "0ba111647c1a6a792997fc6d120b87af91c5bd74a980e6f82dc486138f43dfb5",
|
| 67 |
+
"data/pool/w3-s1": "d17f343e6096fb6f8458fcffe773f86584f795f84710887f822f1a99dc1c1599",
|
| 68 |
+
"data/pool/w3-s2": "a6bfacf6b64cf3a7593eb72eb54a1f7c1467a50a302e4145f0d09317a3e99220",
|
| 69 |
+
"data/pool/w4-s0": "45133b232f854baca586b26947f3e8c5466131936e361fccf164caecd0af2b57",
|
| 70 |
+
"data/pool/w4-s1": "baf14fe23044cc3bb44a06592f7c97dbe7c3841a1b8a4fb8520d142230070786",
|
| 71 |
+
"data/pool/w4-s2": "fefdd4a7b1d055d3075ee7f381a654fc409676648e987dd5cdc6c81ee4419131",
|
| 72 |
+
"data/pool/w5-s0": "467ceab0c667d7fc25a1c46aa6c32a93bc7fe73b9ee772dd418d5cd39c06578a",
|
| 73 |
+
"data/pool/w5-s1": "ec9eff474e21c22c230e4751a853ba5aabd701dcc3954ed01d1ff6343d49d95b",
|
| 74 |
+
"data/pool/w5-s2": "961b61d99794256fda9268f30e77f88e6b62f3bac930536344bd655c3570bff1",
|
| 75 |
+
"data/pool/w6-s0": "0be7dec6c651b282845efbc7a52315ef85919828726e8d6265d9a2daee95e782",
|
| 76 |
+
"data/pool/w6-s1": "c923a29d763af2a7e9b7de4fa884c175ac5ca8b9da380187d5df98c3de0b77d6",
|
| 77 |
+
"data/pool/w6-s2": "3a9229340fd51a3ccce5bff9757ce996ff778bd75e8268a0008be8f333ddfa6e",
|
| 78 |
+
"data/pool/w7-s0": "d9f42180c24d99f7c7489d4e785b816aed0db787ace2995044e8b26799594eee",
|
| 79 |
+
"data/pool/w7-s1": "66e4ce0a96656931b70e219b20938fd899f821399bc320e2fd3a8933189e7d6b",
|
| 80 |
+
"data/pool/w7-s2": "59655904ea2784b39a94561b1ef60d2b7d4012a9089ddf8e6f63b8a38184ec2d"
|
| 81 |
+
}
|
| 82 |
+
}
|
| 83 |
+
}
|
context-multi-bios-20k/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:11424cc903c98513682e9826754421250b863b248d6c8a4bcd37c17c76d40d7b
|
| 3 |
+
size 101882492
|
context-multi-bios-20k/config.json
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "context",
|
| 3 |
+
"data": "data/world-a,data/pool/w3-s0,data/pool/w3-s1,data/pool/w3-s2,data/pool/w4-s0,data/pool/w4-s1,data/pool/w4-s2,data/pool/w5-s0,data/pool/w5-s1,data/pool/w5-s2,data/pool/w6-s0,data/pool/w6-s1,data/pool/w6-s2,data/pool/w7-s0,data/pool/w7-s1,data/pool/w7-s2",
|
| 4 |
+
"out": "runs/context-multi-bios-20k",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"qa",
|
| 13 |
+
0.5
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": null,
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 20000,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.001,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 500,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 2000,
|
| 30 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 31 |
+
"eval_n": 200,
|
| 32 |
+
"save_every": 2000,
|
| 33 |
+
"seed": 0,
|
| 34 |
+
"n_reads": 3,
|
| 35 |
+
"top_k": 4,
|
| 36 |
+
"hop_weight": 0.5,
|
| 37 |
+
"hop_until": 0,
|
| 38 |
+
"max_supervision": 1024,
|
| 39 |
+
"supervise": "all",
|
| 40 |
+
"model_type": "gpt",
|
| 41 |
+
"model": {
|
| 42 |
+
"vocab_size": 576,
|
| 43 |
+
"n_layer": 8,
|
| 44 |
+
"n_head": 8,
|
| 45 |
+
"d_model": 512,
|
| 46 |
+
"mlp_ratio": 4,
|
| 47 |
+
"rope_base": 10000.0
|
| 48 |
+
},
|
| 49 |
+
"params": 25469440,
|
| 50 |
+
"vocab": 544,
|
| 51 |
+
"provenance": {
|
| 52 |
+
"time": "2026-09-25T18:51:19+00:00",
|
| 53 |
+
"code": {
|
| 54 |
+
"commit": "4fc4e676ef82fd2bb4ea3bddc5f4471bc12f3e93",
|
| 55 |
+
"dirty": false
|
| 56 |
+
},
|
| 57 |
+
"env": {
|
| 58 |
+
"python": "3.12.11",
|
| 59 |
+
"platform": "macOS-26.6.2-arm64-arm-64bit",
|
| 60 |
+
"machine": "arm64",
|
| 61 |
+
"mlx": "0.32.2",
|
| 62 |
+
"device": "Device(gpu, 0)"
|
| 63 |
+
},
|
| 64 |
+
"data": {
|
| 65 |
+
"data/world-a": "befab6f0a218e36f48e7ae53539eef4e3fe9fab49eaae9a2d603ee01b86df106",
|
| 66 |
+
"data/pool/w3-s0": "0ba111647c1a6a792997fc6d120b87af91c5bd74a980e6f82dc486138f43dfb5",
|
| 67 |
+
"data/pool/w3-s1": "d17f343e6096fb6f8458fcffe773f86584f795f84710887f822f1a99dc1c1599",
|
| 68 |
+
"data/pool/w3-s2": "a6bfacf6b64cf3a7593eb72eb54a1f7c1467a50a302e4145f0d09317a3e99220",
|
| 69 |
+
"data/pool/w4-s0": "45133b232f854baca586b26947f3e8c5466131936e361fccf164caecd0af2b57",
|
| 70 |
+
"data/pool/w4-s1": "baf14fe23044cc3bb44a06592f7c97dbe7c3841a1b8a4fb8520d142230070786",
|
| 71 |
+
"data/pool/w4-s2": "fefdd4a7b1d055d3075ee7f381a654fc409676648e987dd5cdc6c81ee4419131",
|
| 72 |
+
"data/pool/w5-s0": "467ceab0c667d7fc25a1c46aa6c32a93bc7fe73b9ee772dd418d5cd39c06578a",
|
| 73 |
+
"data/pool/w5-s1": "ec9eff474e21c22c230e4751a853ba5aabd701dcc3954ed01d1ff6343d49d95b",
|
| 74 |
+
"data/pool/w5-s2": "961b61d99794256fda9268f30e77f88e6b62f3bac930536344bd655c3570bff1",
|
| 75 |
+
"data/pool/w6-s0": "0be7dec6c651b282845efbc7a52315ef85919828726e8d6265d9a2daee95e782",
|
| 76 |
+
"data/pool/w6-s1": "c923a29d763af2a7e9b7de4fa884c175ac5ca8b9da380187d5df98c3de0b77d6",
|
| 77 |
+
"data/pool/w6-s2": "3a9229340fd51a3ccce5bff9757ce996ff778bd75e8268a0008be8f333ddfa6e",
|
| 78 |
+
"data/pool/w7-s0": "d9f42180c24d99f7c7489d4e785b816aed0db787ace2995044e8b26799594eee",
|
| 79 |
+
"data/pool/w7-s1": "66e4ce0a96656931b70e219b20938fd899f821399bc320e2fd3a8933189e7d6b",
|
| 80 |
+
"data/pool/w7-s2": "59655904ea2784b39a94561b1ef60d2b7d4012a9089ddf8e6f63b8a38184ec2d"
|
| 81 |
+
}
|
| 82 |
+
}
|
| 83 |
+
}
|
context-multi-bios/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8d97615b89a939b84920786c0b7b6cdd928501886c8fed719a2c3ff239757719
|
| 3 |
+
size 101882492
|
context-multi-bios/config.json
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "context",
|
| 3 |
+
"data": "data/world-a,data/pool/w3-s0,data/pool/w3-s1,data/pool/w3-s2,data/pool/w4-s0,data/pool/w4-s1,data/pool/w4-s2,data/pool/w5-s0,data/pool/w5-s1,data/pool/w5-s2,data/pool/w6-s0,data/pool/w6-s1,data/pool/w6-s2,data/pool/w7-s0,data/pool/w7-s1,data/pool/w7-s2",
|
| 4 |
+
"out": "runs/context-multi-bios",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"qa",
|
| 13 |
+
0.5
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": null,
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 10000,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.001,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 500,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 2000,
|
| 30 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 31 |
+
"eval_n": 200,
|
| 32 |
+
"save_every": 2000,
|
| 33 |
+
"seed": 0,
|
| 34 |
+
"n_reads": 3,
|
| 35 |
+
"top_k": 4,
|
| 36 |
+
"hop_weight": 0.5,
|
| 37 |
+
"hop_until": 0,
|
| 38 |
+
"max_supervision": 1024,
|
| 39 |
+
"supervise": "all",
|
| 40 |
+
"model_type": "gpt",
|
| 41 |
+
"model": {
|
| 42 |
+
"vocab_size": 576,
|
| 43 |
+
"n_layer": 8,
|
| 44 |
+
"n_head": 8,
|
| 45 |
+
"d_model": 512,
|
| 46 |
+
"mlp_ratio": 4,
|
| 47 |
+
"rope_base": 10000.0
|
| 48 |
+
},
|
| 49 |
+
"params": 25469440,
|
| 50 |
+
"vocab": 544,
|
| 51 |
+
"provenance": {
|
| 52 |
+
"time": "2026-09-25T17:18:09+00:00",
|
| 53 |
+
"code": {
|
| 54 |
+
"commit": "4fc4e676ef82fd2bb4ea3bddc5f4471bc12f3e93",
|
| 55 |
+
"dirty": false
|
| 56 |
+
},
|
| 57 |
+
"env": {
|
| 58 |
+
"python": "3.12.11",
|
| 59 |
+
"platform": "macOS-26.6.2-arm64-arm-64bit",
|
| 60 |
+
"machine": "arm64",
|
| 61 |
+
"mlx": "0.32.2",
|
| 62 |
+
"device": "Device(gpu, 0)"
|
| 63 |
+
},
|
| 64 |
+
"data": {
|
| 65 |
+
"data/world-a": "befab6f0a218e36f48e7ae53539eef4e3fe9fab49eaae9a2d603ee01b86df106",
|
| 66 |
+
"data/pool/w3-s0": "0ba111647c1a6a792997fc6d120b87af91c5bd74a980e6f82dc486138f43dfb5",
|
| 67 |
+
"data/pool/w3-s1": "d17f343e6096fb6f8458fcffe773f86584f795f84710887f822f1a99dc1c1599",
|
| 68 |
+
"data/pool/w3-s2": "a6bfacf6b64cf3a7593eb72eb54a1f7c1467a50a302e4145f0d09317a3e99220",
|
| 69 |
+
"data/pool/w4-s0": "45133b232f854baca586b26947f3e8c5466131936e361fccf164caecd0af2b57",
|
| 70 |
+
"data/pool/w4-s1": "baf14fe23044cc3bb44a06592f7c97dbe7c3841a1b8a4fb8520d142230070786",
|
| 71 |
+
"data/pool/w4-s2": "fefdd4a7b1d055d3075ee7f381a654fc409676648e987dd5cdc6c81ee4419131",
|
| 72 |
+
"data/pool/w5-s0": "467ceab0c667d7fc25a1c46aa6c32a93bc7fe73b9ee772dd418d5cd39c06578a",
|
| 73 |
+
"data/pool/w5-s1": "ec9eff474e21c22c230e4751a853ba5aabd701dcc3954ed01d1ff6343d49d95b",
|
| 74 |
+
"data/pool/w5-s2": "961b61d99794256fda9268f30e77f88e6b62f3bac930536344bd655c3570bff1",
|
| 75 |
+
"data/pool/w6-s0": "0be7dec6c651b282845efbc7a52315ef85919828726e8d6265d9a2daee95e782",
|
| 76 |
+
"data/pool/w6-s1": "c923a29d763af2a7e9b7de4fa884c175ac5ca8b9da380187d5df98c3de0b77d6",
|
| 77 |
+
"data/pool/w6-s2": "3a9229340fd51a3ccce5bff9757ce996ff778bd75e8268a0008be8f333ddfa6e",
|
| 78 |
+
"data/pool/w7-s0": "d9f42180c24d99f7c7489d4e785b816aed0db787ace2995044e8b26799594eee",
|
| 79 |
+
"data/pool/w7-s1": "66e4ce0a96656931b70e219b20938fd899f821399bc320e2fd3a8933189e7d6b",
|
| 80 |
+
"data/pool/w7-s2": "59655904ea2784b39a94561b1ef60d2b7d4012a9089ddf8e6f63b8a38184ec2d"
|
| 81 |
+
}
|
| 82 |
+
}
|
| 83 |
+
}
|
dense-a-k100/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c2535e281e08a12dedf5393cd7181be208ff86893980d42800bf627792df93ca
|
| 3 |
+
size 101882492
|
dense-a-k100/config.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "dense",
|
| 3 |
+
"data": "data/world-a",
|
| 4 |
+
"out": "runs/dense-a-k100",
|
| 5 |
+
"edits": 100,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"edit_facts",
|
| 9 |
+
1.0
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"edit_qa",
|
| 13 |
+
1.0
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": "runs/dense-a",
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 200,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.0003,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 20,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 200,
|
| 30 |
+
"eval_sets": "direct,ripple,locality",
|
| 31 |
+
"eval_n": 500,
|
| 32 |
+
"save_every": 0,
|
| 33 |
+
"seed": 0,
|
| 34 |
+
"model": {
|
| 35 |
+
"vocab_size": 576,
|
| 36 |
+
"n_layer": 8,
|
| 37 |
+
"n_head": 8,
|
| 38 |
+
"d_model": 512,
|
| 39 |
+
"mlp_ratio": 4,
|
| 40 |
+
"rope_base": 10000.0
|
| 41 |
+
},
|
| 42 |
+
"params": 25469440,
|
| 43 |
+
"vocab": 544
|
| 44 |
+
}
|
dense-a-to-b/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:17c630e69033c07ee975c8290cce6f41127b2563355ae9bd7b2ecbd6118391c2
|
| 3 |
+
size 101882492
|
dense-a-to-b/config.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "dense",
|
| 3 |
+
"data": "data/world-b",
|
| 4 |
+
"out": "runs/dense-a-to-b",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
1.0
|
| 10 |
+
]
|
| 11 |
+
],
|
| 12 |
+
"init": "runs/dense-a",
|
| 13 |
+
"init_checkpoint": "final",
|
| 14 |
+
"size": "small",
|
| 15 |
+
"dtype": "float32",
|
| 16 |
+
"steps": 2000,
|
| 17 |
+
"batch_size": 64,
|
| 18 |
+
"seq_len": 256,
|
| 19 |
+
"lr": 0.0003,
|
| 20 |
+
"min_lr_frac": 0.1,
|
| 21 |
+
"warmup": 100,
|
| 22 |
+
"weight_decay": 0.1,
|
| 23 |
+
"grad_clip": 1.0,
|
| 24 |
+
"log_every": 100,
|
| 25 |
+
"eval_every": 2000,
|
| 26 |
+
"eval_sets": "all",
|
| 27 |
+
"eval_n": 300,
|
| 28 |
+
"save_every": 0,
|
| 29 |
+
"seed": 0,
|
| 30 |
+
"model": {
|
| 31 |
+
"vocab_size": 576,
|
| 32 |
+
"n_layer": 8,
|
| 33 |
+
"n_head": 8,
|
| 34 |
+
"d_model": 512,
|
| 35 |
+
"mlp_ratio": 4,
|
| 36 |
+
"rope_base": 10000.0
|
| 37 |
+
},
|
| 38 |
+
"params": 25469440,
|
| 39 |
+
"vocab": 544
|
| 40 |
+
}
|
dense-a/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:81e8eb4ac912ffa30fb962977a00297083214f9b75b3bf46a818ef8e058f664b
|
| 3 |
+
size 101882492
|
dense-a/config.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "dense",
|
| 3 |
+
"data": "data/world-a",
|
| 4 |
+
"out": "runs/dense-a",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"qa",
|
| 13 |
+
0.5
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": null,
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 10000,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.001,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 500,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 2000,
|
| 30 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 31 |
+
"eval_n": 200,
|
| 32 |
+
"save_every": 2000,
|
| 33 |
+
"seed": 0,
|
| 34 |
+
"model": {
|
| 35 |
+
"vocab_size": 576,
|
| 36 |
+
"n_layer": 8,
|
| 37 |
+
"n_head": 8,
|
| 38 |
+
"d_model": 512,
|
| 39 |
+
"mlp_ratio": 4,
|
| 40 |
+
"rope_base": 10000.0
|
| 41 |
+
},
|
| 42 |
+
"params": 25469440,
|
| 43 |
+
"vocab": 544
|
| 44 |
+
}
|
latent-a-all/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:57dc7d12714c6cab67037e738530b73f70497f7d2598072be1b7d4471ebcdeaa
|
| 3 |
+
size 116639289
|
latent-a-all/config.json
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "latent",
|
| 3 |
+
"data": "data/world-a",
|
| 4 |
+
"out": "runs/latent-a-all",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"qa",
|
| 13 |
+
0.5
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": null,
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 10000,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.001,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 500,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 2000,
|
| 30 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 31 |
+
"eval_n": 200,
|
| 32 |
+
"save_every": 2000,
|
| 33 |
+
"seed": 0,
|
| 34 |
+
"n_reads": 3,
|
| 35 |
+
"top_k": 4,
|
| 36 |
+
"hop_weight": 0.5,
|
| 37 |
+
"max_supervision": 1024,
|
| 38 |
+
"supervise": "all",
|
| 39 |
+
"model_type": "latent",
|
| 40 |
+
"model": {
|
| 41 |
+
"vocab_size": 576,
|
| 42 |
+
"n_layer": 8,
|
| 43 |
+
"n_head": 8,
|
| 44 |
+
"d_model": 512,
|
| 45 |
+
"mlp_ratio": 4,
|
| 46 |
+
"rope_base": 10000.0,
|
| 47 |
+
"n_reads": 3,
|
| 48 |
+
"top_k": 4,
|
| 49 |
+
"key_dim": 128,
|
| 50 |
+
"retrieval_chunk": 2048,
|
| 51 |
+
"retrieval_group": 256
|
| 52 |
+
},
|
| 53 |
+
"params": 29157915,
|
| 54 |
+
"vocab": 544
|
| 55 |
+
}
|
latent-a/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f4f3b1fbe96a1e08b0cc384a7df6b2e4da7555eb3c4f029206b97845ca42eff0
|
| 3 |
+
size 116639289
|
latent-a/config.json
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "latent",
|
| 3 |
+
"data": "data/world-a",
|
| 4 |
+
"out": "runs/latent-a",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"qa",
|
| 13 |
+
0.5
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": null,
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 10000,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.001,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 500,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 2000,
|
| 30 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 31 |
+
"eval_n": 200,
|
| 32 |
+
"save_every": 2000,
|
| 33 |
+
"seed": 0,
|
| 34 |
+
"n_reads": 3,
|
| 35 |
+
"top_k": 4,
|
| 36 |
+
"hop_weight": 0.5,
|
| 37 |
+
"max_supervision": 512,
|
| 38 |
+
"model_type": "latent",
|
| 39 |
+
"model": {
|
| 40 |
+
"vocab_size": 576,
|
| 41 |
+
"n_layer": 8,
|
| 42 |
+
"n_head": 8,
|
| 43 |
+
"d_model": 512,
|
| 44 |
+
"mlp_ratio": 4,
|
| 45 |
+
"rope_base": 10000.0,
|
| 46 |
+
"n_reads": 3,
|
| 47 |
+
"top_k": 4,
|
| 48 |
+
"key_dim": 128,
|
| 49 |
+
"retrieval_chunk": 2048,
|
| 50 |
+
"retrieval_group": 256
|
| 51 |
+
},
|
| 52 |
+
"params": 29157915,
|
| 53 |
+
"vocab": 544
|
| 54 |
+
}
|
latent-multi-nohop/checkpoints/latest/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dc2494c3833662fbc63b627bd5587866ad0c8d67818ec9325d2f951b9e944259
|
| 3 |
+
size 116639289
|
latent-multi-nohop/config.json
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "latent",
|
| 3 |
+
"data": "data/world-a,data/pool/w3-s0,data/pool/w3-s1,data/pool/w3-s2,data/pool/w4-s0,data/pool/w4-s1,data/pool/w4-s2,data/pool/w5-s0,data/pool/w5-s1,data/pool/w5-s2,data/pool/w6-s0,data/pool/w6-s1,data/pool/w6-s2,data/pool/w7-s0,data/pool/w7-s1,data/pool/w7-s2",
|
| 4 |
+
"out": "runs/latent-multi-nohop",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"qa",
|
| 13 |
+
0.5
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": null,
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 10000,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.001,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 500,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 2000,
|
| 30 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 31 |
+
"eval_n": 200,
|
| 32 |
+
"save_every": 2000,
|
| 33 |
+
"seed": 0,
|
| 34 |
+
"n_reads": 3,
|
| 35 |
+
"top_k": 4,
|
| 36 |
+
"hop_weight": 0.0,
|
| 37 |
+
"max_supervision": 1024,
|
| 38 |
+
"supervise": "all",
|
| 39 |
+
"model_type": "latent",
|
| 40 |
+
"model": {
|
| 41 |
+
"vocab_size": 576,
|
| 42 |
+
"n_layer": 8,
|
| 43 |
+
"n_head": 8,
|
| 44 |
+
"d_model": 512,
|
| 45 |
+
"mlp_ratio": 4,
|
| 46 |
+
"rope_base": 10000.0,
|
| 47 |
+
"n_reads": 3,
|
| 48 |
+
"top_k": 4,
|
| 49 |
+
"key_dim": 128,
|
| 50 |
+
"retrieval_chunk": 2048,
|
| 51 |
+
"retrieval_group": 256
|
| 52 |
+
},
|
| 53 |
+
"params": 29157915,
|
| 54 |
+
"vocab": 544
|
| 55 |
+
}
|
latent-multi/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1b2490e51d93fc40eef41d5e997f714651b1417d52d7d4d2b811833471b51a61
|
| 3 |
+
size 116639289
|
latent-multi/config.json
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "latent",
|
| 3 |
+
"data": "data/world-a,data/pool/w3-s0,data/pool/w3-s1,data/pool/w3-s2,data/pool/w4-s0,data/pool/w4-s1,data/pool/w4-s2,data/pool/w5-s0,data/pool/w5-s1,data/pool/w5-s2,data/pool/w6-s0,data/pool/w6-s1,data/pool/w6-s2,data/pool/w7-s0,data/pool/w7-s1,data/pool/w7-s2",
|
| 4 |
+
"out": "runs/latent-multi",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"qa",
|
| 13 |
+
0.5
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": null,
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 10000,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.001,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 500,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 2000,
|
| 30 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 31 |
+
"eval_n": 200,
|
| 32 |
+
"save_every": 2000,
|
| 33 |
+
"seed": 0,
|
| 34 |
+
"n_reads": 3,
|
| 35 |
+
"top_k": 4,
|
| 36 |
+
"hop_weight": 0.5,
|
| 37 |
+
"max_supervision": 1024,
|
| 38 |
+
"supervise": "all",
|
| 39 |
+
"model_type": "latent",
|
| 40 |
+
"model": {
|
| 41 |
+
"vocab_size": 576,
|
| 42 |
+
"n_layer": 8,
|
| 43 |
+
"n_head": 8,
|
| 44 |
+
"d_model": 512,
|
| 45 |
+
"mlp_ratio": 4,
|
| 46 |
+
"rope_base": 10000.0,
|
| 47 |
+
"n_reads": 3,
|
| 48 |
+
"top_k": 4,
|
| 49 |
+
"key_dim": 128,
|
| 50 |
+
"retrieval_chunk": 2048,
|
| 51 |
+
"retrieval_group": 256
|
| 52 |
+
},
|
| 53 |
+
"params": 29157915,
|
| 54 |
+
"vocab": 544
|
| 55 |
+
}
|
lookup-a/checkpoints/final/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e06385a28712360487c58accc351cf097ef7d1f94c2b2ea6c33c2da1c0f5cae6
|
| 3 |
+
size 101882492
|
lookup-a/config.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"arm": "lookup",
|
| 3 |
+
"data": "data/world-a",
|
| 4 |
+
"out": "runs/lookup-a",
|
| 5 |
+
"edits": 0,
|
| 6 |
+
"mix": [
|
| 7 |
+
[
|
| 8 |
+
"bios",
|
| 9 |
+
0.5
|
| 10 |
+
],
|
| 11 |
+
[
|
| 12 |
+
"qa",
|
| 13 |
+
0.5
|
| 14 |
+
]
|
| 15 |
+
],
|
| 16 |
+
"init": null,
|
| 17 |
+
"init_checkpoint": "final",
|
| 18 |
+
"size": "small",
|
| 19 |
+
"dtype": "float32",
|
| 20 |
+
"steps": 10000,
|
| 21 |
+
"batch_size": 64,
|
| 22 |
+
"seq_len": 256,
|
| 23 |
+
"lr": 0.001,
|
| 24 |
+
"min_lr_frac": 0.1,
|
| 25 |
+
"warmup": 500,
|
| 26 |
+
"weight_decay": 0.1,
|
| 27 |
+
"grad_clip": 1.0,
|
| 28 |
+
"log_every": 100,
|
| 29 |
+
"eval_every": 2000,
|
| 30 |
+
"eval_sets": "test_id,test_ood,test_1hop_ood",
|
| 31 |
+
"eval_n": 200,
|
| 32 |
+
"save_every": 2000,
|
| 33 |
+
"seed": 0,
|
| 34 |
+
"model": {
|
| 35 |
+
"vocab_size": 576,
|
| 36 |
+
"n_layer": 8,
|
| 37 |
+
"n_head": 8,
|
| 38 |
+
"d_model": 512,
|
| 39 |
+
"mlp_ratio": 4,
|
| 40 |
+
"rope_base": 10000.0
|
| 41 |
+
},
|
| 42 |
+
"params": 25469440,
|
| 43 |
+
"vocab": 544
|
| 44 |
+
}
|