Archive verified final models for batch-concentration-pythia70m-v1
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- batch-concentration-pythia70m-v1/README.md +3 -0
- batch-concentration-pythia70m-v1/archive_index.json +0 -0
- batch-concentration-pythia70m-v1/metadata/artifact_index.json +541 -0
- batch-concentration-pythia70m-v1/metadata/resolved_config.json +136 -0
- batch-concentration-pythia70m-v1/metadata/runs.csv +0 -0
- batch-concentration-pythia70m-v1/metadata/runs.lock.json +10 -0
- batch-concentration-pythia70m-v1/metadata/source_lock.json +30 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/config.json +31 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/generation_config.json +7 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/model.safetensors +3 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/provenance.json +25 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/special_tokens_map.json +23 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/tokenizer.json +0 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/tokenizer_config.json +215 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/config.json +31 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/generation_config.json +7 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/model.safetensors +3 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/provenance.json +25 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/special_tokens_map.json +23 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/tokenizer.json +0 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/tokenizer_config.json +215 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/config.json +31 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/generation_config.json +7 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/model.safetensors +3 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/provenance.json +25 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/special_tokens_map.json +23 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/tokenizer.json +0 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/tokenizer_config.json +215 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/config.json +31 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/generation_config.json +7 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/model.safetensors +3 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/provenance.json +25 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/special_tokens_map.json +23 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/tokenizer.json +0 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/tokenizer_config.json +215 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/config.json +31 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/generation_config.json +7 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/model.safetensors +3 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/provenance.json +25 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/special_tokens_map.json +23 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/tokenizer.json +0 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/tokenizer_config.json +215 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/config.json +31 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/generation_config.json +7 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/model.safetensors +3 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/provenance.json +25 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/special_tokens_map.json +23 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/tokenizer.json +0 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/tokenizer_config.json +215 -0
- batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of2-s104729/config.json +31 -0
batch-concentration-pythia70m-v1/README.md
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# batch-concentration-pythia70m-v1
|
| 2 |
+
|
| 3 |
+
Final inference-only checkpoints from the within-update domain-composition study. Each `runs/<run_id>` directory contains Hugging Face model weights, configuration, tokenizer files, and immutable provenance. Optimizer and resume state are intentionally excluded.
|
batch-concentration-pythia70m-v1/archive_index.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-concentration-pythia70m-v1/metadata/artifact_index.json
ADDED
|
@@ -0,0 +1,541 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"artifact_index_hash": "bfa85e55bba66b585cd1ee6a2d4fa7593ae44990cd2dffe7dc5f71c2c3471829",
|
| 3 |
+
"config_hash": "4448409341dab2b8dfb511284f3301aa150d91ff62b747b00014f0ab6a740d08",
|
| 4 |
+
"initializations": {
|
| 5 |
+
"104729": {
|
| 6 |
+
"artifact_sha256": "516e101d79ccc7695e95a0de201c8b264cb7144a8afaa5fc820a4a39ea8a8d43",
|
| 7 |
+
"config_sha256": "66287236cca8a3e712ac5343625a7ec113d1487f31f34d3349037904da4b6e31",
|
| 8 |
+
"model_id": "EleutherAI/pythia-70m",
|
| 9 |
+
"model_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42",
|
| 10 |
+
"parameter_count": 70426624,
|
| 11 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/initializations/104729",
|
| 12 |
+
"seed": 104729,
|
| 13 |
+
"weights_file": "model.safetensors",
|
| 14 |
+
"weights_sha256": "7e3562439edfcdff8c95be2f0bfc66d7eddc10aadd84c1d09718df9043194bb7"
|
| 15 |
+
},
|
| 16 |
+
"130363": {
|
| 17 |
+
"artifact_sha256": "71641e2e8cf4ba87afa6686d49b698065dd2416e6db3102af363fddf10eb60b7",
|
| 18 |
+
"config_sha256": "66287236cca8a3e712ac5343625a7ec113d1487f31f34d3349037904da4b6e31",
|
| 19 |
+
"model_id": "EleutherAI/pythia-70m",
|
| 20 |
+
"model_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42",
|
| 21 |
+
"parameter_count": 70426624,
|
| 22 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/initializations/130363",
|
| 23 |
+
"seed": 130363,
|
| 24 |
+
"weights_file": "model.safetensors",
|
| 25 |
+
"weights_sha256": "c1a9a07a96e56b4aef78f8b65134619bc5fa9d188ab9a930370a21c13e409457"
|
| 26 |
+
},
|
| 27 |
+
"155927": {
|
| 28 |
+
"artifact_sha256": "6d463dd826f71ab8b5c9241d44da39d8319c4f029c42ce2d9733e677cd4ec75b",
|
| 29 |
+
"config_sha256": "66287236cca8a3e712ac5343625a7ec113d1487f31f34d3349037904da4b6e31",
|
| 30 |
+
"model_id": "EleutherAI/pythia-70m",
|
| 31 |
+
"model_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42",
|
| 32 |
+
"parameter_count": 70426624,
|
| 33 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/initializations/155927",
|
| 34 |
+
"seed": 155927,
|
| 35 |
+
"weights_file": "model.safetensors",
|
| 36 |
+
"weights_sha256": "bdba90ce86d558657649958c2d03f4c716f0d81fb6e0efecb4f0bf90f20f6c9a"
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
"orders": {
|
| 40 |
+
"104729": {
|
| 41 |
+
"RedPajamaArXiv": {
|
| 42 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/104729/RedPajamaArXiv.npy",
|
| 43 |
+
"rows": 602112,
|
| 44 |
+
"sha256": "fb510a9c06092fc413790d31daa3026629025e7a2dac72d1f921d8beeed0f63f"
|
| 45 |
+
},
|
| 46 |
+
"RedPajamaBook": {
|
| 47 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/104729/RedPajamaBook.npy",
|
| 48 |
+
"rows": 602112,
|
| 49 |
+
"sha256": "075c170423a579652d643bbac0fc823a2d478f8f7de047da6a6665d12c086b5e"
|
| 50 |
+
},
|
| 51 |
+
"RedPajamaC4": {
|
| 52 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/104729/RedPajamaC4.npy",
|
| 53 |
+
"rows": 602112,
|
| 54 |
+
"sha256": "1e86fba6c1568a9e69eb1cbc5d3f84efe5fee542b3169a7133a5917eab5034b4"
|
| 55 |
+
},
|
| 56 |
+
"RedPajamaGithub": {
|
| 57 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/104729/RedPajamaGithub.npy",
|
| 58 |
+
"rows": 602112,
|
| 59 |
+
"sha256": "482258899489eed5f7db404ca71fb9697e69cf587f168c6cfb86a4b19b00a98c"
|
| 60 |
+
},
|
| 61 |
+
"RedPajamaStackExchange": {
|
| 62 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/104729/RedPajamaStackExchange.npy",
|
| 63 |
+
"rows": 602112,
|
| 64 |
+
"sha256": "11723b01643b3cdfee76c94ce1e92fb88b46c23a4e5699dd674f088da1a68864"
|
| 65 |
+
}
|
| 66 |
+
},
|
| 67 |
+
"130363": {
|
| 68 |
+
"RedPajamaArXiv": {
|
| 69 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/130363/RedPajamaArXiv.npy",
|
| 70 |
+
"rows": 602112,
|
| 71 |
+
"sha256": "2518b0c5260575b07fa7e91f7b8105bfd9f5f33dbc2d25b42fdfcfac40c3320d"
|
| 72 |
+
},
|
| 73 |
+
"RedPajamaBook": {
|
| 74 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/130363/RedPajamaBook.npy",
|
| 75 |
+
"rows": 602112,
|
| 76 |
+
"sha256": "dc2378b907bc0efec91e227c298f4481d1008bc3406cc3e74c2e1db3cc7196e5"
|
| 77 |
+
},
|
| 78 |
+
"RedPajamaC4": {
|
| 79 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/130363/RedPajamaC4.npy",
|
| 80 |
+
"rows": 602112,
|
| 81 |
+
"sha256": "7abf70aeec0c7455364dcac2b59859030f0f575112a09252b3a910f9d07b5ad9"
|
| 82 |
+
},
|
| 83 |
+
"RedPajamaGithub": {
|
| 84 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/130363/RedPajamaGithub.npy",
|
| 85 |
+
"rows": 602112,
|
| 86 |
+
"sha256": "3c33bd5097471a57aa20f7803135de2bdb223d272f11b2b821fdf11bdc11bff8"
|
| 87 |
+
},
|
| 88 |
+
"RedPajamaStackExchange": {
|
| 89 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/130363/RedPajamaStackExchange.npy",
|
| 90 |
+
"rows": 602112,
|
| 91 |
+
"sha256": "9dbd62a1db73e6e1085ca80566f96ee15240634678e013051edb7310337294e2"
|
| 92 |
+
}
|
| 93 |
+
},
|
| 94 |
+
"155927": {
|
| 95 |
+
"RedPajamaArXiv": {
|
| 96 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/155927/RedPajamaArXiv.npy",
|
| 97 |
+
"rows": 602112,
|
| 98 |
+
"sha256": "035ab2c8cea23c6f0e01acd17e7244ea1895f46ee97385f795913ee9c130865f"
|
| 99 |
+
},
|
| 100 |
+
"RedPajamaBook": {
|
| 101 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/155927/RedPajamaBook.npy",
|
| 102 |
+
"rows": 602112,
|
| 103 |
+
"sha256": "d4a63119902232dffb45c8146cd7dce47345d65404f47da54007e62f60b1b916"
|
| 104 |
+
},
|
| 105 |
+
"RedPajamaC4": {
|
| 106 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/155927/RedPajamaC4.npy",
|
| 107 |
+
"rows": 602112,
|
| 108 |
+
"sha256": "ca138015da2f15624c970fd83b259b0f38db739d64a29552854d3112bfb30496"
|
| 109 |
+
},
|
| 110 |
+
"RedPajamaGithub": {
|
| 111 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/155927/RedPajamaGithub.npy",
|
| 112 |
+
"rows": 602112,
|
| 113 |
+
"sha256": "250e551832478d585a39e8f92bb4359d7e5b39e4019f7c5a31d7cb76a9469093"
|
| 114 |
+
},
|
| 115 |
+
"RedPajamaStackExchange": {
|
| 116 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/orders/155927/RedPajamaStackExchange.npy",
|
| 117 |
+
"rows": 602112,
|
| 118 |
+
"sha256": "50e4abcc44e0a3f0043a6f97d3a16dde0b040e632a881786a772ac736c3df3e4"
|
| 119 |
+
}
|
| 120 |
+
}
|
| 121 |
+
},
|
| 122 |
+
"packed": {
|
| 123 |
+
"RedPajamaArXiv": {
|
| 124 |
+
"test": {
|
| 125 |
+
"append_eos_between_documents": true,
|
| 126 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 127 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 128 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaArXiv.documents.sha256.gz",
|
| 129 |
+
"document_hashes_sha256": "2348f1ceff975a5ea0324d404ca2c91e5d49df9495a5f70295c38f60d3fce63a",
|
| 130 |
+
"documents": 17,
|
| 131 |
+
"domain": "RedPajamaArXiv",
|
| 132 |
+
"dtype": "uint16",
|
| 133 |
+
"model_vocab_size": 50304,
|
| 134 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaArXiv.npy",
|
| 135 |
+
"request_hash": "922558e0092c22d8ac44a53c77011301f23f43898486b52db3b1d5b608ec9a32",
|
| 136 |
+
"requested_rows": 128,
|
| 137 |
+
"rows": 128,
|
| 138 |
+
"schema_version": 1,
|
| 139 |
+
"sequence_length": 2048,
|
| 140 |
+
"sha256": "c803c42787bb518a2c00154686aff4a9ea3f7baf1f68c3b248f1dcf8a741c963",
|
| 141 |
+
"split": "test",
|
| 142 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 143 |
+
"tokenizer_maximum_id": 50276,
|
| 144 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 145 |
+
},
|
| 146 |
+
"train": {
|
| 147 |
+
"append_eos_between_documents": true,
|
| 148 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 149 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 150 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaArXiv.documents.sha256.gz",
|
| 151 |
+
"document_hashes_sha256": "39c461bc266982942504b320a8bb2ba700022429ea2a95befe56fe3554f8ef33",
|
| 152 |
+
"documents": 67554,
|
| 153 |
+
"domain": "RedPajamaArXiv",
|
| 154 |
+
"dtype": "uint16",
|
| 155 |
+
"model_vocab_size": 50304,
|
| 156 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaArXiv.npy",
|
| 157 |
+
"request_hash": "31720e56b6f8e32f3ffc964f983cdba4f539aecdc292d521d8c3b4e5f51cdf5d",
|
| 158 |
+
"requested_rows": 602112,
|
| 159 |
+
"rows": 602112,
|
| 160 |
+
"schema_version": 1,
|
| 161 |
+
"sequence_length": 2048,
|
| 162 |
+
"sha256": "2c1bc83a8ffaba4ca02cda72b85f0edc7c1fdd47024b610f73be1312f343416e",
|
| 163 |
+
"split": "train",
|
| 164 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 165 |
+
"tokenizer_maximum_id": 50276,
|
| 166 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 167 |
+
},
|
| 168 |
+
"validation": {
|
| 169 |
+
"append_eos_between_documents": true,
|
| 170 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 171 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 172 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaArXiv.documents.sha256.gz",
|
| 173 |
+
"document_hashes_sha256": "d09db6dbd1e163b5b64d5cddec3d612843a7e9c395fbd9dee2ef7aafbed9e254",
|
| 174 |
+
"documents": 14,
|
| 175 |
+
"domain": "RedPajamaArXiv",
|
| 176 |
+
"dtype": "uint16",
|
| 177 |
+
"model_vocab_size": 50304,
|
| 178 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaArXiv.npy",
|
| 179 |
+
"request_hash": "0fe435ca7e58dcca052d79fbc45a0076d64e102b379bf8e1ed7c720b7cdf56a8",
|
| 180 |
+
"requested_rows": 128,
|
| 181 |
+
"rows": 128,
|
| 182 |
+
"schema_version": 1,
|
| 183 |
+
"sequence_length": 2048,
|
| 184 |
+
"sha256": "1916009eeed999dd13ea2dfedc25ea4c7a70b0437c5b999c26e4a5931a4e9ce2",
|
| 185 |
+
"split": "validation",
|
| 186 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 187 |
+
"tokenizer_maximum_id": 50276,
|
| 188 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 189 |
+
}
|
| 190 |
+
},
|
| 191 |
+
"RedPajamaBook": {
|
| 192 |
+
"test": {
|
| 193 |
+
"append_eos_between_documents": true,
|
| 194 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 195 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 196 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaBook.documents.sha256.gz",
|
| 197 |
+
"document_hashes_sha256": "e181102a6a95187050a94cabce1214a1e475937eddc99cecffb7a591d65b7787",
|
| 198 |
+
"documents": 1,
|
| 199 |
+
"domain": "RedPajamaBook",
|
| 200 |
+
"dtype": "uint16",
|
| 201 |
+
"model_vocab_size": 50304,
|
| 202 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaBook.npy",
|
| 203 |
+
"request_hash": "9bdb0658c0acde11a489091a4f5d8e0b4a3e71a78338e153df18edc3633fb111",
|
| 204 |
+
"requested_rows": 128,
|
| 205 |
+
"rows": 128,
|
| 206 |
+
"schema_version": 1,
|
| 207 |
+
"sequence_length": 2048,
|
| 208 |
+
"sha256": "88ab7f79f51f0b55206e8e5318c2797bbf2c94240575b4b3ddacf6b618d69243",
|
| 209 |
+
"split": "test",
|
| 210 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 211 |
+
"tokenizer_maximum_id": 50276,
|
| 212 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 213 |
+
},
|
| 214 |
+
"train": {
|
| 215 |
+
"append_eos_between_documents": true,
|
| 216 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 217 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 218 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaBook.documents.sha256.gz",
|
| 219 |
+
"document_hashes_sha256": "207d1562a2eec283306aba0407b66b6a227826a964b99c0b5fb700b4a014fad1",
|
| 220 |
+
"documents": 9698,
|
| 221 |
+
"domain": "RedPajamaBook",
|
| 222 |
+
"dtype": "uint16",
|
| 223 |
+
"model_vocab_size": 50304,
|
| 224 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaBook.npy",
|
| 225 |
+
"request_hash": "f0a7ec0e325bebf278f566b6bd5b47341a5a2714dba2a8d33b777c3d42c4f218",
|
| 226 |
+
"requested_rows": 602112,
|
| 227 |
+
"rows": 602112,
|
| 228 |
+
"schema_version": 1,
|
| 229 |
+
"sequence_length": 2048,
|
| 230 |
+
"sha256": "36bcf90b84df4b071cadd41f8d6940217b502c1fd557ce2156d407cd8594ce8d",
|
| 231 |
+
"split": "train",
|
| 232 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 233 |
+
"tokenizer_maximum_id": 50276,
|
| 234 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 235 |
+
},
|
| 236 |
+
"validation": {
|
| 237 |
+
"append_eos_between_documents": true,
|
| 238 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 239 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 240 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaBook.documents.sha256.gz",
|
| 241 |
+
"document_hashes_sha256": "3ddd7ca296f1e81faa9a6776545ff4a3eb169d616c9613e32e606767310fe29b",
|
| 242 |
+
"documents": 3,
|
| 243 |
+
"domain": "RedPajamaBook",
|
| 244 |
+
"dtype": "uint16",
|
| 245 |
+
"model_vocab_size": 50304,
|
| 246 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaBook.npy",
|
| 247 |
+
"request_hash": "c8f0b46411af5898834ae5a851701b2136ed45a9a31e49b7f7f35fd80406bda9",
|
| 248 |
+
"requested_rows": 128,
|
| 249 |
+
"rows": 128,
|
| 250 |
+
"schema_version": 1,
|
| 251 |
+
"sequence_length": 2048,
|
| 252 |
+
"sha256": "d459f761439fa822072b77477c60d1d6013e867299a1523ade538221dcdb86eb",
|
| 253 |
+
"split": "validation",
|
| 254 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 255 |
+
"tokenizer_maximum_id": 50276,
|
| 256 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 257 |
+
}
|
| 258 |
+
},
|
| 259 |
+
"RedPajamaC4": {
|
| 260 |
+
"test": {
|
| 261 |
+
"append_eos_between_documents": true,
|
| 262 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 263 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 264 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaC4.documents.sha256.gz",
|
| 265 |
+
"document_hashes_sha256": "5cb42f240b65c1e7673db67960485991e494634f28f2f5b0e1ea34f6cf5ff6cb",
|
| 266 |
+
"documents": 566,
|
| 267 |
+
"domain": "RedPajamaC4",
|
| 268 |
+
"dtype": "uint16",
|
| 269 |
+
"model_vocab_size": 50304,
|
| 270 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaC4.npy",
|
| 271 |
+
"request_hash": "7decfb6d6ab5f7a1a6becc295182ca842569ac7a5762f7b059e4a6cdb45fc56e",
|
| 272 |
+
"requested_rows": 128,
|
| 273 |
+
"rows": 128,
|
| 274 |
+
"schema_version": 1,
|
| 275 |
+
"sequence_length": 2048,
|
| 276 |
+
"sha256": "e8bf978301f54e51b2b92c6f9608bbcc9ad6b86f191dc6b5a9a09f52293061f3",
|
| 277 |
+
"split": "test",
|
| 278 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 279 |
+
"tokenizer_maximum_id": 50276,
|
| 280 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 281 |
+
},
|
| 282 |
+
"train": {
|
| 283 |
+
"append_eos_between_documents": true,
|
| 284 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 285 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 286 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaC4.documents.sha256.gz",
|
| 287 |
+
"document_hashes_sha256": "8404ae8603fe374007bb61077bbc42c701a061d75f93860311f11d0d964171b0",
|
| 288 |
+
"documents": 2476363,
|
| 289 |
+
"domain": "RedPajamaC4",
|
| 290 |
+
"dtype": "uint16",
|
| 291 |
+
"model_vocab_size": 50304,
|
| 292 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaC4.npy",
|
| 293 |
+
"request_hash": "8c492f477f5e01fe7d4caab260acca7e71809a366240fb92b31cbfa79b70e7fa",
|
| 294 |
+
"requested_rows": 602112,
|
| 295 |
+
"rows": 602112,
|
| 296 |
+
"schema_version": 1,
|
| 297 |
+
"sequence_length": 2048,
|
| 298 |
+
"sha256": "695cac688bf5c45330e56d2564f24873bc0aa17362babf7a6c3d38228bd73d0c",
|
| 299 |
+
"split": "train",
|
| 300 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 301 |
+
"tokenizer_maximum_id": 50276,
|
| 302 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 303 |
+
},
|
| 304 |
+
"validation": {
|
| 305 |
+
"append_eos_between_documents": true,
|
| 306 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 307 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 308 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaC4.documents.sha256.gz",
|
| 309 |
+
"document_hashes_sha256": "05842930c80dd4cf11eb753db0a030164da0b8bde2412b42f5ddd82b2e3ef970",
|
| 310 |
+
"documents": 530,
|
| 311 |
+
"domain": "RedPajamaC4",
|
| 312 |
+
"dtype": "uint16",
|
| 313 |
+
"model_vocab_size": 50304,
|
| 314 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaC4.npy",
|
| 315 |
+
"request_hash": "c27fde416ec4029959e7d5f7fd746ff5cec82279e1c6a36e03ba1220138729f2",
|
| 316 |
+
"requested_rows": 128,
|
| 317 |
+
"rows": 128,
|
| 318 |
+
"schema_version": 1,
|
| 319 |
+
"sequence_length": 2048,
|
| 320 |
+
"sha256": "5356ea0ae1955827ee19f9c89e639e12b2fe6755f7af270dbeee7ac62c085597",
|
| 321 |
+
"split": "validation",
|
| 322 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 323 |
+
"tokenizer_maximum_id": 50276,
|
| 324 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 325 |
+
}
|
| 326 |
+
},
|
| 327 |
+
"RedPajamaGithub": {
|
| 328 |
+
"test": {
|
| 329 |
+
"append_eos_between_documents": true,
|
| 330 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 331 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 332 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaGithub.documents.sha256.gz",
|
| 333 |
+
"document_hashes_sha256": "6dc7b9d4e716cbb16e5cda9fb604c266c3c2fcfd93f8bfe80c8f1da199fe5a7d",
|
| 334 |
+
"documents": 212,
|
| 335 |
+
"domain": "RedPajamaGithub",
|
| 336 |
+
"dtype": "uint16",
|
| 337 |
+
"model_vocab_size": 50304,
|
| 338 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaGithub.npy",
|
| 339 |
+
"request_hash": "74558c71164ce03b3473de2540ff9bb27461008c926d148057801424aa63a8fe",
|
| 340 |
+
"requested_rows": 128,
|
| 341 |
+
"rows": 128,
|
| 342 |
+
"schema_version": 1,
|
| 343 |
+
"sequence_length": 2048,
|
| 344 |
+
"sha256": "b78b206fc90ad3777c754acc2dcd8e9f50fafe4a6e60966c7753bb803fc2f506",
|
| 345 |
+
"split": "test",
|
| 346 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 347 |
+
"tokenizer_maximum_id": 50276,
|
| 348 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 349 |
+
},
|
| 350 |
+
"train": {
|
| 351 |
+
"append_eos_between_documents": true,
|
| 352 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 353 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 354 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaGithub.documents.sha256.gz",
|
| 355 |
+
"document_hashes_sha256": "950bf0b52940b9796bb3f835cc6d752918afb3afac45a18de06b395f5159b4bd",
|
| 356 |
+
"documents": 824299,
|
| 357 |
+
"domain": "RedPajamaGithub",
|
| 358 |
+
"dtype": "uint16",
|
| 359 |
+
"model_vocab_size": 50304,
|
| 360 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaGithub.npy",
|
| 361 |
+
"request_hash": "90d02fe2ed5401c6c511a43d94b57a324725fa425d294f547360a98a5d3ad1e0",
|
| 362 |
+
"requested_rows": 602112,
|
| 363 |
+
"rows": 602112,
|
| 364 |
+
"schema_version": 1,
|
| 365 |
+
"sequence_length": 2048,
|
| 366 |
+
"sha256": "dc03f0470cd5e3a892bbc80e1124d7a4cc9a8e069936350b72cd242453ff0ce0",
|
| 367 |
+
"split": "train",
|
| 368 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 369 |
+
"tokenizer_maximum_id": 50276,
|
| 370 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 371 |
+
},
|
| 372 |
+
"validation": {
|
| 373 |
+
"append_eos_between_documents": true,
|
| 374 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 375 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 376 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaGithub.documents.sha256.gz",
|
| 377 |
+
"document_hashes_sha256": "3ae356cdace0d01a4eeb9973036c88c88a3701c13635c277d944036f138abc29",
|
| 378 |
+
"documents": 196,
|
| 379 |
+
"domain": "RedPajamaGithub",
|
| 380 |
+
"dtype": "uint16",
|
| 381 |
+
"model_vocab_size": 50304,
|
| 382 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaGithub.npy",
|
| 383 |
+
"request_hash": "5ca9255cae0f1de14bc33ef1881017368f9bfff4f141625598ea5b4b4ee27aae",
|
| 384 |
+
"requested_rows": 128,
|
| 385 |
+
"rows": 128,
|
| 386 |
+
"schema_version": 1,
|
| 387 |
+
"sequence_length": 2048,
|
| 388 |
+
"sha256": "177f2d413385dfcac6a5480aee18c8395a83abff2ccd2fa52061b44e28013493",
|
| 389 |
+
"split": "validation",
|
| 390 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 391 |
+
"tokenizer_maximum_id": 50276,
|
| 392 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 393 |
+
}
|
| 394 |
+
},
|
| 395 |
+
"RedPajamaStackExchange": {
|
| 396 |
+
"test": {
|
| 397 |
+
"append_eos_between_documents": true,
|
| 398 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 399 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 400 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaStackExchange.documents.sha256.gz",
|
| 401 |
+
"document_hashes_sha256": "d8188f200ab1e906ec53f3281dbfeac2921ea1128eb6aea048d64840b6b9238e",
|
| 402 |
+
"documents": 380,
|
| 403 |
+
"domain": "RedPajamaStackExchange",
|
| 404 |
+
"dtype": "uint16",
|
| 405 |
+
"model_vocab_size": 50304,
|
| 406 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/test/RedPajamaStackExchange.npy",
|
| 407 |
+
"request_hash": "fb0a081e2daeb21e472c11d921b38ec5f78fc594826f8744591dd5b0d3d2f317",
|
| 408 |
+
"requested_rows": 128,
|
| 409 |
+
"rows": 128,
|
| 410 |
+
"schema_version": 1,
|
| 411 |
+
"sequence_length": 2048,
|
| 412 |
+
"sha256": "b365ef282ca7b613a1e1c283e271cc47fc5ecd29df0bd90e49f61ed829725dea",
|
| 413 |
+
"split": "test",
|
| 414 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 415 |
+
"tokenizer_maximum_id": 50276,
|
| 416 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 417 |
+
},
|
| 418 |
+
"train": {
|
| 419 |
+
"append_eos_between_documents": true,
|
| 420 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 421 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 422 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaStackExchange.documents.sha256.gz",
|
| 423 |
+
"document_hashes_sha256": "5121d98b057469948f2195faa3ec2587d255ca9ed179f202e91330408630195b",
|
| 424 |
+
"documents": 1786596,
|
| 425 |
+
"domain": "RedPajamaStackExchange",
|
| 426 |
+
"dtype": "uint16",
|
| 427 |
+
"model_vocab_size": 50304,
|
| 428 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/train/RedPajamaStackExchange.npy",
|
| 429 |
+
"request_hash": "a649ee6f433ffb1882374a157efb26fb837b3fcf4305ac96e59163fb7f7406b3",
|
| 430 |
+
"requested_rows": 602112,
|
| 431 |
+
"rows": 602112,
|
| 432 |
+
"schema_version": 1,
|
| 433 |
+
"sequence_length": 2048,
|
| 434 |
+
"sha256": "920cc45cd6ea708c8cc89800049aebaeb2ab428a9f5b358bfacff8817ed0f88d",
|
| 435 |
+
"split": "train",
|
| 436 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 437 |
+
"tokenizer_maximum_id": 50276,
|
| 438 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 439 |
+
},
|
| 440 |
+
"validation": {
|
| 441 |
+
"append_eos_between_documents": true,
|
| 442 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 443 |
+
"dataset_revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36",
|
| 444 |
+
"document_hashes_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaStackExchange.documents.sha256.gz",
|
| 445 |
+
"document_hashes_sha256": "1b60880986814d07aeb0e39a7d9e6843c81ab8c49110423cfdbbfb889553cd1c",
|
| 446 |
+
"documents": 422,
|
| 447 |
+
"domain": "RedPajamaStackExchange",
|
| 448 |
+
"dtype": "uint16",
|
| 449 |
+
"model_vocab_size": 50304,
|
| 450 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/packed/validation/RedPajamaStackExchange.npy",
|
| 451 |
+
"request_hash": "3070168ded123bd5487e7497050db55fd7a86e39ce45c2059083bf3564404a0b",
|
| 452 |
+
"requested_rows": 128,
|
| 453 |
+
"rows": 128,
|
| 454 |
+
"schema_version": 1,
|
| 455 |
+
"sequence_length": 2048,
|
| 456 |
+
"sha256": "81d1d54968a467b4a4e18c19b16bd63aac7ee2b490c1b34e000f77ffd53b062e",
|
| 457 |
+
"split": "validation",
|
| 458 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 459 |
+
"tokenizer_maximum_id": 50276,
|
| 460 |
+
"tokenizer_revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 461 |
+
}
|
| 462 |
+
}
|
| 463 |
+
},
|
| 464 |
+
"schedules": {
|
| 465 |
+
"104729": {
|
| 466 |
+
"1/2": {
|
| 467 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/schedules/104729/p-1of2.npy",
|
| 468 |
+
"phase": 1,
|
| 469 |
+
"proportion": "1/2",
|
| 470 |
+
"rows": 5376,
|
| 471 |
+
"sha256": "981200f392906726674f25a9c27c14d74bbde656d72e320a1d3f85cf5bcf8f0f"
|
| 472 |
+
},
|
| 473 |
+
"1/8": {
|
| 474 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/schedules/104729/p-1of8.npy",
|
| 475 |
+
"phase": 1,
|
| 476 |
+
"proportion": "1/8",
|
| 477 |
+
"rows": 5376,
|
| 478 |
+
"sha256": "637a6cea2c7e9dc9a0918fb54475f7314848113b6189c722c567db8ac902e307"
|
| 479 |
+
},
|
| 480 |
+
"7/8": {
|
| 481 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/schedules/104729/p-7of8.npy",
|
| 482 |
+
"phase": 1,
|
| 483 |
+
"proportion": "7/8",
|
| 484 |
+
"rows": 5376,
|
| 485 |
+
"sha256": "25c27975b9fa282739875f995edcf7e63d0e829ec0a218ece1a51fceb243c704"
|
| 486 |
+
}
|
| 487 |
+
},
|
| 488 |
+
"130363": {
|
| 489 |
+
"1/2": {
|
| 490 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/schedules/130363/p-1of2.npy",
|
| 491 |
+
"phase": 3,
|
| 492 |
+
"proportion": "1/2",
|
| 493 |
+
"rows": 5376,
|
| 494 |
+
"sha256": "981200f392906726674f25a9c27c14d74bbde656d72e320a1d3f85cf5bcf8f0f"
|
| 495 |
+
},
|
| 496 |
+
"1/8": {
|
| 497 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/schedules/130363/p-1of8.npy",
|
| 498 |
+
"phase": 3,
|
| 499 |
+
"proportion": "1/8",
|
| 500 |
+
"rows": 5376,
|
| 501 |
+
"sha256": "5acf8dc6097d0419fed512b44ee5051599bb6a81fae9614ea3328acf468e6d9e"
|
| 502 |
+
},
|
| 503 |
+
"7/8": {
|
| 504 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/schedules/130363/p-7of8.npy",
|
| 505 |
+
"phase": 3,
|
| 506 |
+
"proportion": "7/8",
|
| 507 |
+
"rows": 5376,
|
| 508 |
+
"sha256": "ba93e482a3a31afbf4dc812b996a53e29fc4cd0b7f1031575657d101766f1d43"
|
| 509 |
+
}
|
| 510 |
+
},
|
| 511 |
+
"155927": {
|
| 512 |
+
"1/2": {
|
| 513 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/schedules/155927/p-1of2.npy",
|
| 514 |
+
"phase": 7,
|
| 515 |
+
"proportion": "1/2",
|
| 516 |
+
"rows": 5376,
|
| 517 |
+
"sha256": "981200f392906726674f25a9c27c14d74bbde656d72e320a1d3f85cf5bcf8f0f"
|
| 518 |
+
},
|
| 519 |
+
"1/8": {
|
| 520 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/schedules/155927/p-1of8.npy",
|
| 521 |
+
"phase": 7,
|
| 522 |
+
"proportion": "1/8",
|
| 523 |
+
"rows": 5376,
|
| 524 |
+
"sha256": "98cee3ad1606c5a9c4f7a064795bbc2b3dbbe68b35e0680e6b77ce52b6bd08e6"
|
| 525 |
+
},
|
| 526 |
+
"7/8": {
|
| 527 |
+
"path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/schedules/155927/p-7of8.npy",
|
| 528 |
+
"phase": 7,
|
| 529 |
+
"proportion": "7/8",
|
| 530 |
+
"rows": 5376,
|
| 531 |
+
"sha256": "9378ea4c0f4793e43573de5ad1c573947e73b00cd9194edbc5159d0340d21a23"
|
| 532 |
+
}
|
| 533 |
+
}
|
| 534 |
+
},
|
| 535 |
+
"schema_version": 1,
|
| 536 |
+
"source_lock": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/source_lock.json",
|
| 537 |
+
"source_lock_hash": "1948b4af225df5ec8119bdc2ed0af661a98020770595ca4adb8fe9d09594760c",
|
| 538 |
+
"study": "batch-concentration-pythia70m-v1",
|
| 539 |
+
"tokenizer_files_hash": "99a09a6737d7ec985e78217e9d96785cf36cf5164ceb6c477b56962615203cee",
|
| 540 |
+
"tokenizer_path": "/lfs/skampere2/0/l1ly/batch-concentration/data/batch-concentration-pythia70m-v1/tokenizer"
|
| 541 |
+
}
|
batch-concentration-pythia70m-v1/metadata/resolved_config.json
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"analysis": {
|
| 3 |
+
"adaptive_seed_extension": false,
|
| 4 |
+
"confidence_level": 0.95,
|
| 5 |
+
"equivalence_margin_nats": 0.01,
|
| 6 |
+
"interval_method": "paired_t",
|
| 7 |
+
"primary_split": "test",
|
| 8 |
+
"require_all_cells": true,
|
| 9 |
+
"sensitivity_summary": "rms_nonzero_alpha_delta"
|
| 10 |
+
},
|
| 11 |
+
"data": {
|
| 12 |
+
"append_eos_between_documents": true,
|
| 13 |
+
"probe_sequences_per_domain": 16,
|
| 14 |
+
"reuse_domain_stream_across_pairs": true,
|
| 15 |
+
"test_sequences_per_domain": 128,
|
| 16 |
+
"test_split": "test",
|
| 17 |
+
"train_split": "train",
|
| 18 |
+
"validation_sequences_per_domain": 128,
|
| 19 |
+
"validation_split": "validation"
|
| 20 |
+
},
|
| 21 |
+
"design": {
|
| 22 |
+
"allowed_cells": null,
|
| 23 |
+
"concentrations": [
|
| 24 |
+
"0",
|
| 25 |
+
"1/4",
|
| 26 |
+
"1/2",
|
| 27 |
+
"3/4",
|
| 28 |
+
"1"
|
| 29 |
+
],
|
| 30 |
+
"focus_period": 8,
|
| 31 |
+
"global_batch_sequences": 128,
|
| 32 |
+
"master_seeds": [
|
| 33 |
+
104729,
|
| 34 |
+
130363,
|
| 35 |
+
155927
|
| 36 |
+
],
|
| 37 |
+
"optimizer_updates": 5376,
|
| 38 |
+
"pairs": [
|
| 39 |
+
{
|
| 40 |
+
"first": "RedPajamaArXiv",
|
| 41 |
+
"id": "arxiv_stackexchange",
|
| 42 |
+
"second": "RedPajamaStackExchange"
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"first": "RedPajamaGithub",
|
| 46 |
+
"id": "github_c4",
|
| 47 |
+
"second": "RedPajamaC4"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"first": "RedPajamaBook",
|
| 51 |
+
"id": "books_stackexchange",
|
| 52 |
+
"second": "RedPajamaStackExchange"
|
| 53 |
+
}
|
| 54 |
+
],
|
| 55 |
+
"proportions": [
|
| 56 |
+
"1/8",
|
| 57 |
+
"1/2",
|
| 58 |
+
"7/8"
|
| 59 |
+
],
|
| 60 |
+
"sequence_length": 2048
|
| 61 |
+
},
|
| 62 |
+
"execution": {
|
| 63 |
+
"checkpoint_keep_last": 2,
|
| 64 |
+
"stale_claim_hours": 12,
|
| 65 |
+
"transient_retry_limit": 2
|
| 66 |
+
},
|
| 67 |
+
"model": {
|
| 68 |
+
"architecture": "pythia-70m",
|
| 69 |
+
"attention_backend": "sdpa",
|
| 70 |
+
"gradient_checkpointing": false,
|
| 71 |
+
"zero_dropout": true
|
| 72 |
+
},
|
| 73 |
+
"paths": {
|
| 74 |
+
"data_root": "${CONCENTRATION_DATA_ROOT}",
|
| 75 |
+
"output_root": "outputs",
|
| 76 |
+
"run_root": "${CONCENTRATION_RUN_ROOT}"
|
| 77 |
+
},
|
| 78 |
+
"schema_version": 1,
|
| 79 |
+
"sources": {
|
| 80 |
+
"dataset_id": "gmongaras/SlimPajama-627B_Reupload",
|
| 81 |
+
"dataset_revision": "resolve",
|
| 82 |
+
"model_id": "EleutherAI/pythia-70m",
|
| 83 |
+
"model_revision": "resolve",
|
| 84 |
+
"tokenizer_id": "EleutherAI/pythia-70m",
|
| 85 |
+
"tokenizer_revision": "resolve"
|
| 86 |
+
},
|
| 87 |
+
"study": {
|
| 88 |
+
"kind": "primary",
|
| 89 |
+
"manifest_seed": 8675309,
|
| 90 |
+
"name": "batch-concentration-pythia70m-v1",
|
| 91 |
+
"status": "frozen"
|
| 92 |
+
},
|
| 93 |
+
"tracking": {
|
| 94 |
+
"mode": "online",
|
| 95 |
+
"wandb_entity": null,
|
| 96 |
+
"wandb_project": "batch-concentration-pretraining"
|
| 97 |
+
},
|
| 98 |
+
"training": {
|
| 99 |
+
"betas": [
|
| 100 |
+
0.9,
|
| 101 |
+
0.95
|
| 102 |
+
],
|
| 103 |
+
"checkpoint_updates": [
|
| 104 |
+
1024,
|
| 105 |
+
2048,
|
| 106 |
+
3072,
|
| 107 |
+
4096,
|
| 108 |
+
5120,
|
| 109 |
+
5376
|
| 110 |
+
],
|
| 111 |
+
"epsilon": 1e-08,
|
| 112 |
+
"evaluation_updates": [
|
| 113 |
+
0,
|
| 114 |
+
512,
|
| 115 |
+
1024,
|
| 116 |
+
2048,
|
| 117 |
+
4096,
|
| 118 |
+
5376
|
| 119 |
+
],
|
| 120 |
+
"fused_optimizer": true,
|
| 121 |
+
"gradient_clip_norm": 1.0,
|
| 122 |
+
"gradient_probe_updates": [
|
| 123 |
+
0,
|
| 124 |
+
5376
|
| 125 |
+
],
|
| 126 |
+
"max_learning_rate": 0.001,
|
| 127 |
+
"micro_batch_sequences": 16,
|
| 128 |
+
"optimizer": "adamw",
|
| 129 |
+
"precision": "bf16",
|
| 130 |
+
"schedule": "cosine_to_zero",
|
| 131 |
+
"telemetry_every_updates": 64,
|
| 132 |
+
"update_norm_every_updates": 512,
|
| 133 |
+
"warmup_updates": 64,
|
| 134 |
+
"weight_decay": 0.1
|
| 135 |
+
}
|
| 136 |
+
}
|
batch-concentration-pythia70m-v1/metadata/runs.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-concentration-pythia70m-v1/metadata/runs.lock.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"artifact_index_hash": "bfa85e55bba66b585cd1ee6a2d4fa7593ae44990cd2dffe7dc5f71c2c3471829",
|
| 3 |
+
"config_hash": "4448409341dab2b8dfb511284f3301aa150d91ff62b747b00014f0ab6a740d08",
|
| 4 |
+
"implementation_hash": "0424f42e02e5ad0963fa4319af2b2dd91dcd63de7df9ea7753a999dc932b52d4",
|
| 5 |
+
"manifest_sha256": "ac36ff4c869e9e0eed4dcde9e7ba4c72b20f1119088f2b10352870428cfb07b7",
|
| 6 |
+
"rows": 135,
|
| 7 |
+
"schema_version": 1,
|
| 8 |
+
"source_lock_hash": "1948b4af225df5ec8119bdc2ed0af661a98020770595ca4adb8fe9d09594760c",
|
| 9 |
+
"study": "batch-concentration-pythia70m-v1"
|
| 10 |
+
}
|
batch-concentration-pythia70m-v1/metadata/source_lock.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"dataset": {
|
| 3 |
+
"id": "gmongaras/SlimPajama-627B_Reupload",
|
| 4 |
+
"revision": "c34c22dbb10ae6b264a2f357a909d1a537141b36"
|
| 5 |
+
},
|
| 6 |
+
"lock_hash": "e2f017166adaa1d4691045e6919bdab2a7bf0cd6362beadfd4ac0528a415c441",
|
| 7 |
+
"model": {
|
| 8 |
+
"id": "EleutherAI/pythia-70m",
|
| 9 |
+
"revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 10 |
+
},
|
| 11 |
+
"platform": "Linux-6.8.0-117-generic-x86_64-with-glibc2.39",
|
| 12 |
+
"python": "3.12.3 (main, Jan 22 2026, 20:57:42) [GCC 13.3.0]",
|
| 13 |
+
"schema_version": 1,
|
| 14 |
+
"software_at_resolution": {
|
| 15 |
+
"accelerate": "1.11.0",
|
| 16 |
+
"datasets": "4.2.0",
|
| 17 |
+
"huggingface-hub": "0.35.3",
|
| 18 |
+
"numpy": "1.26.4",
|
| 19 |
+
"pyarrow": "22.0.0",
|
| 20 |
+
"pydantic": "2.12.0",
|
| 21 |
+
"safetensors": "0.6.2",
|
| 22 |
+
"torch": "2.6.0+cu124",
|
| 23 |
+
"transformers": "4.57.1",
|
| 24 |
+
"wandb": "0.28.1"
|
| 25 |
+
},
|
| 26 |
+
"tokenizer": {
|
| 27 |
+
"id": "EleutherAI/pythia-70m",
|
| 28 |
+
"revision": "a39f36b100fe8a5377810d56c3f4789b9c53ac42"
|
| 29 |
+
}
|
| 30 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/config.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"GPTNeoXForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": true,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"bos_token_id": 0,
|
| 8 |
+
"classifier_dropout": 0.0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"eos_token_id": 0,
|
| 11 |
+
"hidden_act": "gelu",
|
| 12 |
+
"hidden_dropout": 0.0,
|
| 13 |
+
"hidden_size": 512,
|
| 14 |
+
"initializer_range": 0.02,
|
| 15 |
+
"intermediate_size": 2048,
|
| 16 |
+
"layer_norm_eps": 1e-05,
|
| 17 |
+
"max_position_embeddings": 2048,
|
| 18 |
+
"model_type": "gpt_neox",
|
| 19 |
+
"num_attention_heads": 8,
|
| 20 |
+
"num_hidden_layers": 6,
|
| 21 |
+
"partial_rotary_factor": 0.25,
|
| 22 |
+
"rope_scaling": null,
|
| 23 |
+
"rope_theta": 10000,
|
| 24 |
+
"rotary_emb_base": 10000,
|
| 25 |
+
"rotary_pct": 0.25,
|
| 26 |
+
"tie_word_embeddings": false,
|
| 27 |
+
"transformers_version": "4.57.1",
|
| 28 |
+
"use_cache": false,
|
| 29 |
+
"use_parallel_residual": true,
|
| 30 |
+
"vocab_size": 50304
|
| 31 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/generation_config.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"transformers_version": "4.57.1",
|
| 6 |
+
"use_cache": false
|
| 7 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:282991a500d2bd8ae60085f96b5ab963ffff45c8e31a5dcf3d221c5b20ef39e7
|
| 3 |
+
size 281715176
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/provenance.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"artifact_index_hash": "bfa85e55bba66b585cd1ee6a2d4fa7593ae44990cd2dffe7dc5f71c2c3471829",
|
| 3 |
+
"completed_updates": 5376,
|
| 4 |
+
"concentration": "0",
|
| 5 |
+
"config_hash": "4448409341dab2b8dfb511284f3301aa150d91ff62b747b00014f0ab6a740d08",
|
| 6 |
+
"domain_first": "RedPajamaArXiv",
|
| 7 |
+
"domain_second": "RedPajamaStackExchange",
|
| 8 |
+
"final_metrics": {
|
| 9 |
+
"test/first_nll": 2.188092201948166,
|
| 10 |
+
"test/macro_nll": 2.374615952372551,
|
| 11 |
+
"test/second_nll": 2.561139702796936,
|
| 12 |
+
"test/weighted_nll": 2.374615952372551,
|
| 13 |
+
"validation/first_nll": 2.044421523809433,
|
| 14 |
+
"validation/macro_nll": 2.2258306592702866,
|
| 15 |
+
"validation/second_nll": 2.40723979473114,
|
| 16 |
+
"validation/weighted_nll": 2.2258306592702866
|
| 17 |
+
},
|
| 18 |
+
"initialization_hash": "7e3562439edfcdff8c95be2f0bfc66d7eddc10aadd84c1d09718df9043194bb7",
|
| 19 |
+
"manifest_hash": "ac36ff4c869e9e0eed4dcde9e7ba4c72b20f1119088f2b10352870428cfb07b7",
|
| 20 |
+
"pair_id": "arxiv_stackexchange",
|
| 21 |
+
"proportion": "1/2",
|
| 22 |
+
"run_id": "arxiv_stackexchange-p1of2-a0of1-s104729",
|
| 23 |
+
"seed": "104729",
|
| 24 |
+
"source_lock_hash": "1948b4af225df5ec8119bdc2ed0af661a98020770595ca4adb8fe9d09594760c"
|
| 25 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<|endoftext|>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "<|endoftext|>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"unk_token": {
|
| 17 |
+
"content": "<|endoftext|>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s104729/tokenizer_config.json
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
}
|
| 206 |
+
},
|
| 207 |
+
"bos_token": "<|endoftext|>",
|
| 208 |
+
"clean_up_tokenization_spaces": false,
|
| 209 |
+
"eos_token": "<|endoftext|>",
|
| 210 |
+
"extra_special_tokens": {},
|
| 211 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 212 |
+
"pad_token": null,
|
| 213 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 214 |
+
"unk_token": "<|endoftext|>"
|
| 215 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/config.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"GPTNeoXForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": true,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"bos_token_id": 0,
|
| 8 |
+
"classifier_dropout": 0.0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"eos_token_id": 0,
|
| 11 |
+
"hidden_act": "gelu",
|
| 12 |
+
"hidden_dropout": 0.0,
|
| 13 |
+
"hidden_size": 512,
|
| 14 |
+
"initializer_range": 0.02,
|
| 15 |
+
"intermediate_size": 2048,
|
| 16 |
+
"layer_norm_eps": 1e-05,
|
| 17 |
+
"max_position_embeddings": 2048,
|
| 18 |
+
"model_type": "gpt_neox",
|
| 19 |
+
"num_attention_heads": 8,
|
| 20 |
+
"num_hidden_layers": 6,
|
| 21 |
+
"partial_rotary_factor": 0.25,
|
| 22 |
+
"rope_scaling": null,
|
| 23 |
+
"rope_theta": 10000,
|
| 24 |
+
"rotary_emb_base": 10000,
|
| 25 |
+
"rotary_pct": 0.25,
|
| 26 |
+
"tie_word_embeddings": false,
|
| 27 |
+
"transformers_version": "4.57.1",
|
| 28 |
+
"use_cache": false,
|
| 29 |
+
"use_parallel_residual": true,
|
| 30 |
+
"vocab_size": 50304
|
| 31 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/generation_config.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"transformers_version": "4.57.1",
|
| 6 |
+
"use_cache": false
|
| 7 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1f7ce984b284abc58f883dab91bded775c6190cc79b46bdccd41014955fb3ba0
|
| 3 |
+
size 281715176
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/provenance.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"artifact_index_hash": "bfa85e55bba66b585cd1ee6a2d4fa7593ae44990cd2dffe7dc5f71c2c3471829",
|
| 3 |
+
"completed_updates": 5376,
|
| 4 |
+
"concentration": "0",
|
| 5 |
+
"config_hash": "4448409341dab2b8dfb511284f3301aa150d91ff62b747b00014f0ab6a740d08",
|
| 6 |
+
"domain_first": "RedPajamaArXiv",
|
| 7 |
+
"domain_second": "RedPajamaStackExchange",
|
| 8 |
+
"final_metrics": {
|
| 9 |
+
"test/first_nll": 2.1789613366127014,
|
| 10 |
+
"test/macro_nll": 2.3659945875406265,
|
| 11 |
+
"test/second_nll": 2.5530278384685516,
|
| 12 |
+
"test/weighted_nll": 2.3659945875406265,
|
| 13 |
+
"validation/first_nll": 2.0320166051387787,
|
| 14 |
+
"validation/macro_nll": 2.216390371322632,
|
| 15 |
+
"validation/second_nll": 2.400764137506485,
|
| 16 |
+
"validation/weighted_nll": 2.216390371322632
|
| 17 |
+
},
|
| 18 |
+
"initialization_hash": "c1a9a07a96e56b4aef78f8b65134619bc5fa9d188ab9a930370a21c13e409457",
|
| 19 |
+
"manifest_hash": "ac36ff4c869e9e0eed4dcde9e7ba4c72b20f1119088f2b10352870428cfb07b7",
|
| 20 |
+
"pair_id": "arxiv_stackexchange",
|
| 21 |
+
"proportion": "1/2",
|
| 22 |
+
"run_id": "arxiv_stackexchange-p1of2-a0of1-s130363",
|
| 23 |
+
"seed": "130363",
|
| 24 |
+
"source_lock_hash": "1948b4af225df5ec8119bdc2ed0af661a98020770595ca4adb8fe9d09594760c"
|
| 25 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<|endoftext|>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "<|endoftext|>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"unk_token": {
|
| 17 |
+
"content": "<|endoftext|>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s130363/tokenizer_config.json
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
}
|
| 206 |
+
},
|
| 207 |
+
"bos_token": "<|endoftext|>",
|
| 208 |
+
"clean_up_tokenization_spaces": false,
|
| 209 |
+
"eos_token": "<|endoftext|>",
|
| 210 |
+
"extra_special_tokens": {},
|
| 211 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 212 |
+
"pad_token": null,
|
| 213 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 214 |
+
"unk_token": "<|endoftext|>"
|
| 215 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/config.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"GPTNeoXForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": true,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"bos_token_id": 0,
|
| 8 |
+
"classifier_dropout": 0.0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"eos_token_id": 0,
|
| 11 |
+
"hidden_act": "gelu",
|
| 12 |
+
"hidden_dropout": 0.0,
|
| 13 |
+
"hidden_size": 512,
|
| 14 |
+
"initializer_range": 0.02,
|
| 15 |
+
"intermediate_size": 2048,
|
| 16 |
+
"layer_norm_eps": 1e-05,
|
| 17 |
+
"max_position_embeddings": 2048,
|
| 18 |
+
"model_type": "gpt_neox",
|
| 19 |
+
"num_attention_heads": 8,
|
| 20 |
+
"num_hidden_layers": 6,
|
| 21 |
+
"partial_rotary_factor": 0.25,
|
| 22 |
+
"rope_scaling": null,
|
| 23 |
+
"rope_theta": 10000,
|
| 24 |
+
"rotary_emb_base": 10000,
|
| 25 |
+
"rotary_pct": 0.25,
|
| 26 |
+
"tie_word_embeddings": false,
|
| 27 |
+
"transformers_version": "4.57.1",
|
| 28 |
+
"use_cache": false,
|
| 29 |
+
"use_parallel_residual": true,
|
| 30 |
+
"vocab_size": 50304
|
| 31 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/generation_config.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"transformers_version": "4.57.1",
|
| 6 |
+
"use_cache": false
|
| 7 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ec4efe728e8bd2f92393aed0d55ff785aceba18e094a2a1d22aa3155691558b7
|
| 3 |
+
size 281715176
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/provenance.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"artifact_index_hash": "bfa85e55bba66b585cd1ee6a2d4fa7593ae44990cd2dffe7dc5f71c2c3471829",
|
| 3 |
+
"completed_updates": 5376,
|
| 4 |
+
"concentration": "0",
|
| 5 |
+
"config_hash": "4448409341dab2b8dfb511284f3301aa150d91ff62b747b00014f0ab6a740d08",
|
| 6 |
+
"domain_first": "RedPajamaArXiv",
|
| 7 |
+
"domain_second": "RedPajamaStackExchange",
|
| 8 |
+
"final_metrics": {
|
| 9 |
+
"test/first_nll": 2.178909942507744,
|
| 10 |
+
"test/macro_nll": 2.3648932203650475,
|
| 11 |
+
"test/second_nll": 2.550876498222351,
|
| 12 |
+
"test/weighted_nll": 2.3648932203650475,
|
| 13 |
+
"validation/first_nll": 2.0352811366319656,
|
| 14 |
+
"validation/macro_nll": 2.219103641808033,
|
| 15 |
+
"validation/second_nll": 2.4029261469841003,
|
| 16 |
+
"validation/weighted_nll": 2.219103641808033
|
| 17 |
+
},
|
| 18 |
+
"initialization_hash": "bdba90ce86d558657649958c2d03f4c716f0d81fb6e0efecb4f0bf90f20f6c9a",
|
| 19 |
+
"manifest_hash": "ac36ff4c869e9e0eed4dcde9e7ba4c72b20f1119088f2b10352870428cfb07b7",
|
| 20 |
+
"pair_id": "arxiv_stackexchange",
|
| 21 |
+
"proportion": "1/2",
|
| 22 |
+
"run_id": "arxiv_stackexchange-p1of2-a0of1-s155927",
|
| 23 |
+
"seed": "155927",
|
| 24 |
+
"source_lock_hash": "1948b4af225df5ec8119bdc2ed0af661a98020770595ca4adb8fe9d09594760c"
|
| 25 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<|endoftext|>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "<|endoftext|>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"unk_token": {
|
| 17 |
+
"content": "<|endoftext|>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a0of1-s155927/tokenizer_config.json
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
}
|
| 206 |
+
},
|
| 207 |
+
"bos_token": "<|endoftext|>",
|
| 208 |
+
"clean_up_tokenization_spaces": false,
|
| 209 |
+
"eos_token": "<|endoftext|>",
|
| 210 |
+
"extra_special_tokens": {},
|
| 211 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 212 |
+
"pad_token": null,
|
| 213 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 214 |
+
"unk_token": "<|endoftext|>"
|
| 215 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/config.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"GPTNeoXForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": true,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"bos_token_id": 0,
|
| 8 |
+
"classifier_dropout": 0.0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"eos_token_id": 0,
|
| 11 |
+
"hidden_act": "gelu",
|
| 12 |
+
"hidden_dropout": 0.0,
|
| 13 |
+
"hidden_size": 512,
|
| 14 |
+
"initializer_range": 0.02,
|
| 15 |
+
"intermediate_size": 2048,
|
| 16 |
+
"layer_norm_eps": 1e-05,
|
| 17 |
+
"max_position_embeddings": 2048,
|
| 18 |
+
"model_type": "gpt_neox",
|
| 19 |
+
"num_attention_heads": 8,
|
| 20 |
+
"num_hidden_layers": 6,
|
| 21 |
+
"partial_rotary_factor": 0.25,
|
| 22 |
+
"rope_scaling": null,
|
| 23 |
+
"rope_theta": 10000,
|
| 24 |
+
"rotary_emb_base": 10000,
|
| 25 |
+
"rotary_pct": 0.25,
|
| 26 |
+
"tie_word_embeddings": false,
|
| 27 |
+
"transformers_version": "4.57.1",
|
| 28 |
+
"use_cache": false,
|
| 29 |
+
"use_parallel_residual": true,
|
| 30 |
+
"vocab_size": 50304
|
| 31 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/generation_config.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"transformers_version": "4.57.1",
|
| 6 |
+
"use_cache": false
|
| 7 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:34f73ad68eab4967677138205f83f375322721fb6ba66f2e21bc88cb0d4ea306
|
| 3 |
+
size 281715176
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/provenance.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"artifact_index_hash": "bfa85e55bba66b585cd1ee6a2d4fa7593ae44990cd2dffe7dc5f71c2c3471829",
|
| 3 |
+
"completed_updates": 5376,
|
| 4 |
+
"concentration": "1",
|
| 5 |
+
"config_hash": "4448409341dab2b8dfb511284f3301aa150d91ff62b747b00014f0ab6a740d08",
|
| 6 |
+
"domain_first": "RedPajamaArXiv",
|
| 7 |
+
"domain_second": "RedPajamaStackExchange",
|
| 8 |
+
"final_metrics": {
|
| 9 |
+
"test/first_nll": 2.194272592663765,
|
| 10 |
+
"test/macro_nll": 2.383065663278103,
|
| 11 |
+
"test/second_nll": 2.571858733892441,
|
| 12 |
+
"test/weighted_nll": 2.383065663278103,
|
| 13 |
+
"validation/first_nll": 2.046232834458351,
|
| 14 |
+
"validation/macro_nll": 2.2323748394846916,
|
| 15 |
+
"validation/second_nll": 2.418516844511032,
|
| 16 |
+
"validation/weighted_nll": 2.2323748394846916
|
| 17 |
+
},
|
| 18 |
+
"initialization_hash": "7e3562439edfcdff8c95be2f0bfc66d7eddc10aadd84c1d09718df9043194bb7",
|
| 19 |
+
"manifest_hash": "ac36ff4c869e9e0eed4dcde9e7ba4c72b20f1119088f2b10352870428cfb07b7",
|
| 20 |
+
"pair_id": "arxiv_stackexchange",
|
| 21 |
+
"proportion": "1/2",
|
| 22 |
+
"run_id": "arxiv_stackexchange-p1of2-a1of1-s104729",
|
| 23 |
+
"seed": "104729",
|
| 24 |
+
"source_lock_hash": "1948b4af225df5ec8119bdc2ed0af661a98020770595ca4adb8fe9d09594760c"
|
| 25 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<|endoftext|>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "<|endoftext|>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"unk_token": {
|
| 17 |
+
"content": "<|endoftext|>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s104729/tokenizer_config.json
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
}
|
| 206 |
+
},
|
| 207 |
+
"bos_token": "<|endoftext|>",
|
| 208 |
+
"clean_up_tokenization_spaces": false,
|
| 209 |
+
"eos_token": "<|endoftext|>",
|
| 210 |
+
"extra_special_tokens": {},
|
| 211 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 212 |
+
"pad_token": null,
|
| 213 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 214 |
+
"unk_token": "<|endoftext|>"
|
| 215 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/config.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"GPTNeoXForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": true,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"bos_token_id": 0,
|
| 8 |
+
"classifier_dropout": 0.0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"eos_token_id": 0,
|
| 11 |
+
"hidden_act": "gelu",
|
| 12 |
+
"hidden_dropout": 0.0,
|
| 13 |
+
"hidden_size": 512,
|
| 14 |
+
"initializer_range": 0.02,
|
| 15 |
+
"intermediate_size": 2048,
|
| 16 |
+
"layer_norm_eps": 1e-05,
|
| 17 |
+
"max_position_embeddings": 2048,
|
| 18 |
+
"model_type": "gpt_neox",
|
| 19 |
+
"num_attention_heads": 8,
|
| 20 |
+
"num_hidden_layers": 6,
|
| 21 |
+
"partial_rotary_factor": 0.25,
|
| 22 |
+
"rope_scaling": null,
|
| 23 |
+
"rope_theta": 10000,
|
| 24 |
+
"rotary_emb_base": 10000,
|
| 25 |
+
"rotary_pct": 0.25,
|
| 26 |
+
"tie_word_embeddings": false,
|
| 27 |
+
"transformers_version": "4.57.1",
|
| 28 |
+
"use_cache": false,
|
| 29 |
+
"use_parallel_residual": true,
|
| 30 |
+
"vocab_size": 50304
|
| 31 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/generation_config.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"transformers_version": "4.57.1",
|
| 6 |
+
"use_cache": false
|
| 7 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:29f85270aa42de20598256605e59fe23444a311035b9995bfd04e3edcc01770d
|
| 3 |
+
size 281715176
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/provenance.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"artifact_index_hash": "bfa85e55bba66b585cd1ee6a2d4fa7593ae44990cd2dffe7dc5f71c2c3471829",
|
| 3 |
+
"completed_updates": 5376,
|
| 4 |
+
"concentration": "1",
|
| 5 |
+
"config_hash": "4448409341dab2b8dfb511284f3301aa150d91ff62b747b00014f0ab6a740d08",
|
| 6 |
+
"domain_first": "RedPajamaArXiv",
|
| 7 |
+
"domain_second": "RedPajamaStackExchange",
|
| 8 |
+
"final_metrics": {
|
| 9 |
+
"test/first_nll": 2.2081029564142227,
|
| 10 |
+
"test/macro_nll": 2.3988688066601753,
|
| 11 |
+
"test/second_nll": 2.589634656906128,
|
| 12 |
+
"test/weighted_nll": 2.3988688066601753,
|
| 13 |
+
"validation/first_nll": 2.061548203229904,
|
| 14 |
+
"validation/macro_nll": 2.2503587305545807,
|
| 15 |
+
"validation/second_nll": 2.439169257879257,
|
| 16 |
+
"validation/weighted_nll": 2.2503587305545807
|
| 17 |
+
},
|
| 18 |
+
"initialization_hash": "c1a9a07a96e56b4aef78f8b65134619bc5fa9d188ab9a930370a21c13e409457",
|
| 19 |
+
"manifest_hash": "ac36ff4c869e9e0eed4dcde9e7ba4c72b20f1119088f2b10352870428cfb07b7",
|
| 20 |
+
"pair_id": "arxiv_stackexchange",
|
| 21 |
+
"proportion": "1/2",
|
| 22 |
+
"run_id": "arxiv_stackexchange-p1of2-a1of1-s130363",
|
| 23 |
+
"seed": "130363",
|
| 24 |
+
"source_lock_hash": "1948b4af225df5ec8119bdc2ed0af661a98020770595ca4adb8fe9d09594760c"
|
| 25 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<|endoftext|>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "<|endoftext|>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"unk_token": {
|
| 17 |
+
"content": "<|endoftext|>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s130363/tokenizer_config.json
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
}
|
| 206 |
+
},
|
| 207 |
+
"bos_token": "<|endoftext|>",
|
| 208 |
+
"clean_up_tokenization_spaces": false,
|
| 209 |
+
"eos_token": "<|endoftext|>",
|
| 210 |
+
"extra_special_tokens": {},
|
| 211 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 212 |
+
"pad_token": null,
|
| 213 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 214 |
+
"unk_token": "<|endoftext|>"
|
| 215 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/config.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"GPTNeoXForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": true,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"bos_token_id": 0,
|
| 8 |
+
"classifier_dropout": 0.0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"eos_token_id": 0,
|
| 11 |
+
"hidden_act": "gelu",
|
| 12 |
+
"hidden_dropout": 0.0,
|
| 13 |
+
"hidden_size": 512,
|
| 14 |
+
"initializer_range": 0.02,
|
| 15 |
+
"intermediate_size": 2048,
|
| 16 |
+
"layer_norm_eps": 1e-05,
|
| 17 |
+
"max_position_embeddings": 2048,
|
| 18 |
+
"model_type": "gpt_neox",
|
| 19 |
+
"num_attention_heads": 8,
|
| 20 |
+
"num_hidden_layers": 6,
|
| 21 |
+
"partial_rotary_factor": 0.25,
|
| 22 |
+
"rope_scaling": null,
|
| 23 |
+
"rope_theta": 10000,
|
| 24 |
+
"rotary_emb_base": 10000,
|
| 25 |
+
"rotary_pct": 0.25,
|
| 26 |
+
"tie_word_embeddings": false,
|
| 27 |
+
"transformers_version": "4.57.1",
|
| 28 |
+
"use_cache": false,
|
| 29 |
+
"use_parallel_residual": true,
|
| 30 |
+
"vocab_size": 50304
|
| 31 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/generation_config.json
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"bos_token_id": 0,
|
| 4 |
+
"eos_token_id": 0,
|
| 5 |
+
"transformers_version": "4.57.1",
|
| 6 |
+
"use_cache": false
|
| 7 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:431e5f7d01dbd718837d356642a4215fb1927e180609de4f2d2cf3e0cb7f3896
|
| 3 |
+
size 281715176
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/provenance.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"artifact_index_hash": "bfa85e55bba66b585cd1ee6a2d4fa7593ae44990cd2dffe7dc5f71c2c3471829",
|
| 3 |
+
"completed_updates": 5376,
|
| 4 |
+
"concentration": "1",
|
| 5 |
+
"config_hash": "4448409341dab2b8dfb511284f3301aa150d91ff62b747b00014f0ab6a740d08",
|
| 6 |
+
"domain_first": "RedPajamaArXiv",
|
| 7 |
+
"domain_second": "RedPajamaStackExchange",
|
| 8 |
+
"final_metrics": {
|
| 9 |
+
"test/first_nll": 2.1987888514995575,
|
| 10 |
+
"test/macro_nll": 2.391088530421257,
|
| 11 |
+
"test/second_nll": 2.5833882093429565,
|
| 12 |
+
"test/weighted_nll": 2.391088530421257,
|
| 13 |
+
"validation/first_nll": 2.0557921230793,
|
| 14 |
+
"validation/macro_nll": 2.2437224239110947,
|
| 15 |
+
"validation/second_nll": 2.4316527247428894,
|
| 16 |
+
"validation/weighted_nll": 2.2437224239110947
|
| 17 |
+
},
|
| 18 |
+
"initialization_hash": "bdba90ce86d558657649958c2d03f4c716f0d81fb6e0efecb4f0bf90f20f6c9a",
|
| 19 |
+
"manifest_hash": "ac36ff4c869e9e0eed4dcde9e7ba4c72b20f1119088f2b10352870428cfb07b7",
|
| 20 |
+
"pair_id": "arxiv_stackexchange",
|
| 21 |
+
"proportion": "1/2",
|
| 22 |
+
"run_id": "arxiv_stackexchange-p1of2-a1of1-s155927",
|
| 23 |
+
"seed": "155927",
|
| 24 |
+
"source_lock_hash": "1948b4af225df5ec8119bdc2ed0af661a98020770595ca4adb8fe9d09594760c"
|
| 25 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/special_tokens_map.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token": {
|
| 3 |
+
"content": "<|endoftext|>",
|
| 4 |
+
"lstrip": false,
|
| 5 |
+
"normalized": false,
|
| 6 |
+
"rstrip": false,
|
| 7 |
+
"single_word": false
|
| 8 |
+
},
|
| 9 |
+
"eos_token": {
|
| 10 |
+
"content": "<|endoftext|>",
|
| 11 |
+
"lstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"rstrip": false,
|
| 14 |
+
"single_word": false
|
| 15 |
+
},
|
| 16 |
+
"unk_token": {
|
| 17 |
+
"content": "<|endoftext|>",
|
| 18 |
+
"lstrip": false,
|
| 19 |
+
"normalized": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"single_word": false
|
| 22 |
+
}
|
| 23 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of1-s155927/tokenizer_config.json
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": false,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"add_prefix_space": false,
|
| 5 |
+
"added_tokens_decoder": {
|
| 6 |
+
"0": {
|
| 7 |
+
"content": "<|endoftext|>",
|
| 8 |
+
"lstrip": false,
|
| 9 |
+
"normalized": false,
|
| 10 |
+
"rstrip": false,
|
| 11 |
+
"single_word": false,
|
| 12 |
+
"special": true
|
| 13 |
+
},
|
| 14 |
+
"1": {
|
| 15 |
+
"content": "<|padding|>",
|
| 16 |
+
"lstrip": false,
|
| 17 |
+
"normalized": false,
|
| 18 |
+
"rstrip": false,
|
| 19 |
+
"single_word": false,
|
| 20 |
+
"special": true
|
| 21 |
+
},
|
| 22 |
+
"50254": {
|
| 23 |
+
"content": " ",
|
| 24 |
+
"lstrip": false,
|
| 25 |
+
"normalized": true,
|
| 26 |
+
"rstrip": false,
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"special": false
|
| 29 |
+
},
|
| 30 |
+
"50255": {
|
| 31 |
+
"content": " ",
|
| 32 |
+
"lstrip": false,
|
| 33 |
+
"normalized": true,
|
| 34 |
+
"rstrip": false,
|
| 35 |
+
"single_word": false,
|
| 36 |
+
"special": false
|
| 37 |
+
},
|
| 38 |
+
"50256": {
|
| 39 |
+
"content": " ",
|
| 40 |
+
"lstrip": false,
|
| 41 |
+
"normalized": true,
|
| 42 |
+
"rstrip": false,
|
| 43 |
+
"single_word": false,
|
| 44 |
+
"special": false
|
| 45 |
+
},
|
| 46 |
+
"50257": {
|
| 47 |
+
"content": " ",
|
| 48 |
+
"lstrip": false,
|
| 49 |
+
"normalized": true,
|
| 50 |
+
"rstrip": false,
|
| 51 |
+
"single_word": false,
|
| 52 |
+
"special": false
|
| 53 |
+
},
|
| 54 |
+
"50258": {
|
| 55 |
+
"content": " ",
|
| 56 |
+
"lstrip": false,
|
| 57 |
+
"normalized": true,
|
| 58 |
+
"rstrip": false,
|
| 59 |
+
"single_word": false,
|
| 60 |
+
"special": false
|
| 61 |
+
},
|
| 62 |
+
"50259": {
|
| 63 |
+
"content": " ",
|
| 64 |
+
"lstrip": false,
|
| 65 |
+
"normalized": true,
|
| 66 |
+
"rstrip": false,
|
| 67 |
+
"single_word": false,
|
| 68 |
+
"special": false
|
| 69 |
+
},
|
| 70 |
+
"50260": {
|
| 71 |
+
"content": " ",
|
| 72 |
+
"lstrip": false,
|
| 73 |
+
"normalized": true,
|
| 74 |
+
"rstrip": false,
|
| 75 |
+
"single_word": false,
|
| 76 |
+
"special": false
|
| 77 |
+
},
|
| 78 |
+
"50261": {
|
| 79 |
+
"content": " ",
|
| 80 |
+
"lstrip": false,
|
| 81 |
+
"normalized": true,
|
| 82 |
+
"rstrip": false,
|
| 83 |
+
"single_word": false,
|
| 84 |
+
"special": false
|
| 85 |
+
},
|
| 86 |
+
"50262": {
|
| 87 |
+
"content": " ",
|
| 88 |
+
"lstrip": false,
|
| 89 |
+
"normalized": true,
|
| 90 |
+
"rstrip": false,
|
| 91 |
+
"single_word": false,
|
| 92 |
+
"special": false
|
| 93 |
+
},
|
| 94 |
+
"50263": {
|
| 95 |
+
"content": " ",
|
| 96 |
+
"lstrip": false,
|
| 97 |
+
"normalized": true,
|
| 98 |
+
"rstrip": false,
|
| 99 |
+
"single_word": false,
|
| 100 |
+
"special": false
|
| 101 |
+
},
|
| 102 |
+
"50264": {
|
| 103 |
+
"content": " ",
|
| 104 |
+
"lstrip": false,
|
| 105 |
+
"normalized": true,
|
| 106 |
+
"rstrip": false,
|
| 107 |
+
"single_word": false,
|
| 108 |
+
"special": false
|
| 109 |
+
},
|
| 110 |
+
"50265": {
|
| 111 |
+
"content": " ",
|
| 112 |
+
"lstrip": false,
|
| 113 |
+
"normalized": true,
|
| 114 |
+
"rstrip": false,
|
| 115 |
+
"single_word": false,
|
| 116 |
+
"special": false
|
| 117 |
+
},
|
| 118 |
+
"50266": {
|
| 119 |
+
"content": " ",
|
| 120 |
+
"lstrip": false,
|
| 121 |
+
"normalized": true,
|
| 122 |
+
"rstrip": false,
|
| 123 |
+
"single_word": false,
|
| 124 |
+
"special": false
|
| 125 |
+
},
|
| 126 |
+
"50267": {
|
| 127 |
+
"content": " ",
|
| 128 |
+
"lstrip": false,
|
| 129 |
+
"normalized": true,
|
| 130 |
+
"rstrip": false,
|
| 131 |
+
"single_word": false,
|
| 132 |
+
"special": false
|
| 133 |
+
},
|
| 134 |
+
"50268": {
|
| 135 |
+
"content": " ",
|
| 136 |
+
"lstrip": false,
|
| 137 |
+
"normalized": true,
|
| 138 |
+
"rstrip": false,
|
| 139 |
+
"single_word": false,
|
| 140 |
+
"special": false
|
| 141 |
+
},
|
| 142 |
+
"50269": {
|
| 143 |
+
"content": " ",
|
| 144 |
+
"lstrip": false,
|
| 145 |
+
"normalized": true,
|
| 146 |
+
"rstrip": false,
|
| 147 |
+
"single_word": false,
|
| 148 |
+
"special": false
|
| 149 |
+
},
|
| 150 |
+
"50270": {
|
| 151 |
+
"content": " ",
|
| 152 |
+
"lstrip": false,
|
| 153 |
+
"normalized": true,
|
| 154 |
+
"rstrip": false,
|
| 155 |
+
"single_word": false,
|
| 156 |
+
"special": false
|
| 157 |
+
},
|
| 158 |
+
"50271": {
|
| 159 |
+
"content": " ",
|
| 160 |
+
"lstrip": false,
|
| 161 |
+
"normalized": true,
|
| 162 |
+
"rstrip": false,
|
| 163 |
+
"single_word": false,
|
| 164 |
+
"special": false
|
| 165 |
+
},
|
| 166 |
+
"50272": {
|
| 167 |
+
"content": " ",
|
| 168 |
+
"lstrip": false,
|
| 169 |
+
"normalized": true,
|
| 170 |
+
"rstrip": false,
|
| 171 |
+
"single_word": false,
|
| 172 |
+
"special": false
|
| 173 |
+
},
|
| 174 |
+
"50273": {
|
| 175 |
+
"content": " ",
|
| 176 |
+
"lstrip": false,
|
| 177 |
+
"normalized": true,
|
| 178 |
+
"rstrip": false,
|
| 179 |
+
"single_word": false,
|
| 180 |
+
"special": false
|
| 181 |
+
},
|
| 182 |
+
"50274": {
|
| 183 |
+
"content": " ",
|
| 184 |
+
"lstrip": false,
|
| 185 |
+
"normalized": true,
|
| 186 |
+
"rstrip": false,
|
| 187 |
+
"single_word": false,
|
| 188 |
+
"special": false
|
| 189 |
+
},
|
| 190 |
+
"50275": {
|
| 191 |
+
"content": " ",
|
| 192 |
+
"lstrip": false,
|
| 193 |
+
"normalized": true,
|
| 194 |
+
"rstrip": false,
|
| 195 |
+
"single_word": false,
|
| 196 |
+
"special": false
|
| 197 |
+
},
|
| 198 |
+
"50276": {
|
| 199 |
+
"content": " ",
|
| 200 |
+
"lstrip": false,
|
| 201 |
+
"normalized": true,
|
| 202 |
+
"rstrip": false,
|
| 203 |
+
"single_word": false,
|
| 204 |
+
"special": false
|
| 205 |
+
}
|
| 206 |
+
},
|
| 207 |
+
"bos_token": "<|endoftext|>",
|
| 208 |
+
"clean_up_tokenization_spaces": false,
|
| 209 |
+
"eos_token": "<|endoftext|>",
|
| 210 |
+
"extra_special_tokens": {},
|
| 211 |
+
"model_max_length": 1000000000000000019884624838656,
|
| 212 |
+
"pad_token": null,
|
| 213 |
+
"tokenizer_class": "GPTNeoXTokenizer",
|
| 214 |
+
"unk_token": "<|endoftext|>"
|
| 215 |
+
}
|
batch-concentration-pythia70m-v1/runs/arxiv_stackexchange-p1of2-a1of2-s104729/config.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"GPTNeoXForCausalLM"
|
| 4 |
+
],
|
| 5 |
+
"attention_bias": true,
|
| 6 |
+
"attention_dropout": 0.0,
|
| 7 |
+
"bos_token_id": 0,
|
| 8 |
+
"classifier_dropout": 0.0,
|
| 9 |
+
"dtype": "float32",
|
| 10 |
+
"eos_token_id": 0,
|
| 11 |
+
"hidden_act": "gelu",
|
| 12 |
+
"hidden_dropout": 0.0,
|
| 13 |
+
"hidden_size": 512,
|
| 14 |
+
"initializer_range": 0.02,
|
| 15 |
+
"intermediate_size": 2048,
|
| 16 |
+
"layer_norm_eps": 1e-05,
|
| 17 |
+
"max_position_embeddings": 2048,
|
| 18 |
+
"model_type": "gpt_neox",
|
| 19 |
+
"num_attention_heads": 8,
|
| 20 |
+
"num_hidden_layers": 6,
|
| 21 |
+
"partial_rotary_factor": 0.25,
|
| 22 |
+
"rope_scaling": null,
|
| 23 |
+
"rope_theta": 10000,
|
| 24 |
+
"rotary_emb_base": 10000,
|
| 25 |
+
"rotary_pct": 0.25,
|
| 26 |
+
"tie_word_embeddings": false,
|
| 27 |
+
"transformers_version": "4.57.1",
|
| 28 |
+
"use_cache": false,
|
| 29 |
+
"use_parallel_residual": true,
|
| 30 |
+
"vocab_size": 50304
|
| 31 |
+
}
|