Add models

README.md

@@ -1,3 +1,65 @@
 ---
 license: apache-2.0
 ---
+
+# Interchangeable Token Embeddings
+
+This repository contains the models for the ICML 2025 paper, [Interchangeable Token Embeddings for Extendable Vocabulary and Alpha-Equivalence](https://arxiv.org/abs/2410.17161).
+
+Links:
+1. [arXiv](https://arxiv.org/abs/2410.17161)
+2. [Project Page](https://necrashter.github.io/interchangeable-token-embeddings/)
+3. [Code](https://github.com/necrashter/interchangeable-token-embeddings)
+
+## Models
+
+Figure 3a (LTL):
+| Model | Path |
+| ----- | ---- |
+| Proposed Method | `models/ltl-5/d005-rop-bn1-fn1-ada1-s42` |
+| Baseline | `models/adacos/nondyn-rope` |
+| Full-Vocabulary | `models/10ap/adacos-nondyn` |
+| Alpha-Renaming | `models/ltl-5/s010-rop-bn1-fn1-ada1-s46` |
+
+Figure 3b (Propositional Logic):
+| Model | Path |
+| ----- | ---- |
+| Proposed Method | `models-prop/5ap/d020-rop-bn1-fn1-ada1-s44` |
+| Baseline | `models-prop/5ap/0000-rop-bn0-fn1-ada1-s42` |
+| Full-Vocabulary | `models-prop/10ap/0000-rop-bn0-fn1-ada1-s43` |
+| Alpha-Renaming | `models-prop/5ap/s010-rop-bn1-fn1-ada1-s44` |
+
+Figure 5a (Ablation - LTL):
+| Model | Path |
+| ----- | ---- |
+| w/o Adacos | `models/ltl-5/d005-rop-bn1-fn1-ada0-s42` |
+| w/o $f_{{fn}}$ & AdaCos | `models/ltl-5/d005-rop-bn1-fn0-ada0-s42` |
+| w/o $f_{{bn}}$ | `models/ltl-5/d005-rop-bn0-fn1-ada1-s42` |
+
+Figure 5b (Ablation - Propositional Logic):
+| Model | Path |
+| ----- | ---- |
+| w/o Adacos | `models-prop/5ap/d020-rop-bn1-fn1-ada0-s43` |
+| w/o $f_{{fn}}$ & AdaCos | `models-prop/5ap/d020-rop-bn1-fn0-ada0-s44` |
+| w/o $f_{{bn}}$ | `models-prop/5ap/d020-rop-bn0-fn1-ada1-s42` |
+
+### Perturbation Experiments
+
+Table 2 and 7 (LTL):
+| Model | Path |
+| ----- | ---- |
+| Normal Baseline | `models/adacos/nondyn-rope` |
+| Perturbed Baseline | `models/adacos/nondyn-rope-perturbed` |
+| Perturbed Alpha-Renaming | `models/adacos/s005-rope-perturbed-s42` |
+| Perturbed Proposed | `models/adacos/diagbor4-rope-perturbed` |
+| Limited Baseline | `models/adacos/nondyn-rope-limited` |
+| Limited Alpha-Renaming | `models/adacos/s005-rope-limited-s42` |
+| Limited Proposed | `models/adacos/diagbor4-rope-limited` |
+
+Table 8 (Propositional Logic):
+| Model | Path |
+| ----- | ---- |
+| Normal Baseline | `models-prop/5ap/0000-rop-bn0-fn1-ada1-s42` |
+| Perturbed Baseline | `models-prop/5ap/base-rop-bn0-fn1-ada1-s44` |
+| Perturbed Alpha-Renaming | `models-prop/5ap/s005-rop-bn1-fn1-ada1-s43` |
+| Perturbed Proposed | `models-prop/5ap/d020-rop-bn1-fn1-ada1-s42` |

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/command-log.txt

@@ -0,0 +1,67 @@
+Number of parameters: 2_904_336
+Arguments:
+{
+    "model_path": "models-prop/10ap/0000-rop-bn0-fn1-ada1-s43",
+    "device": "cuda",
+    "seed": 43,
+    "subparser": "train-ted",
+    "data_dir": "data-prop",
+    "ds_name": "ltl-35-10ap",
+    "max_trace_length": -1,
+    "min_aps": null,
+    "max_aps": null,
+    "exact_aps": null,
+    "vocab_aps": null,
+    "val_split": "val",
+    "merged_vocab": true,
+    "merge_tokens": "all",
+    "dynamic_aps": false,
+    "learning_rate": 0.001,
+    "lr_scheduler_type": "cosine",
+    "warmup_steps": 1000,
+    "weight_decay": 0.1,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.95,
+    "max_grad_norm": 1.0,
+    "epochs": 64,
+    "batch_size": 1024,
+    "grad_acc_steps": 1,
+    "logging_steps": 500,
+    "eval_steps": 3000,
+    "train_max_samples": null,
+    "val_max_samples": 10000,
+    "trace_max_samples": 100,
+    "dry": false,
+    "eval": false,
+    "resume": false,
+    "loss_fct": "adacos",
+    "num_heads": 6,
+    "d_embed_enc": 132,
+    "d_embed_dec": null,
+    "d_ff": 512,
+    "ff_activation": "relu",
+    "num_layers": 6,
+    "dropout": 0.1,
+    "layer_norm_eps": 1e-06,
+    "enc_pe": "sinusoid",
+    "dec_pe": "rope",
+    "no_pe_cross_keys": false,
+    "tree_pos_enc": true,
+    "d_ap": 0,
+    "ap_embed": "randn",
+    "embed_base_normalization": "disabled",
+    "embed_ap_normalization": "disabled",
+    "embed_final_normalization": "l2",
+    "feature_normalization": "l2",
+    "embed_scaling": "sqrtd",
+    "shuffle_aps": null,
+    "action": "train",
+    "model_type": "ted",
+    "decoder_only": false
+}
+Using CUDA device: NVIDIA H100
+Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]
+numpy version: 1.26.2
+torch version: 2.1.1+cu121
+transformers version: 4.39.3
+

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/config.json

@@ -0,0 +1,60 @@
+{
+    "vocab": {
+        "aps": [
+            "a",
+            "b",
+            "c",
+            "d",
+            "e",
+            "f",
+            "g",
+            "h",
+            "i",
+            "j"
+        ],
+        "consts": [
+            "0",
+            "1"
+        ],
+        "trace_ops": [],
+        "ltl_ops": [
+            "!",
+            "&",
+            "|",
+            "=",
+            "^"
+        ],
+        "merge_tokens": "all",
+        "dynamic_aps": false,
+        "use_start_token": true,
+        "use_pad_token": true,
+        "use_eos_token": true
+    },
+    "d_embed_enc": 132,
+    "d_embed_dec": 132,
+    "d_ff": 512,
+    "ff_activation": "relu",
+    "dropout": 0.1,
+    "num_heads": 6,
+    "num_layers": 6,
+    "layer_norm_eps": 1e-06,
+    "merged_embedder": {
+        "tie_embeddings": true,
+        "pad_vocab_size_multiple": 8,
+        "d_ap": 0,
+        "ap_embed": "randn",
+        "base_normalization": "disabled",
+        "ap_normalization": "disabled",
+        "final_normalization": "l2",
+        "feature_normalization": "l2",
+        "embed_scaling": "sqrtd",
+        "shuffle_aps": null
+    },
+    "max_encode_length": 1024,
+    "max_decode_length": 1024,
+    "tree_pos_enc": true,
+    "datatype": "float32",
+    "enc_pe": "sinusoid",
+    "dec_pe": "rope",
+    "no_pe_cross_keys": false
+}

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ae22f58ed2164dbda8a13f51c013c5c39e50583c3415e795d5c017144064187
+size 2682581

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da1.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4772a6943f13f92b797149e7e6f206c3ee4c411cf7681fd32fc7be3f669330ae
+size 3218204

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91f72633f1aa61db03ec9e1aeeee6e2179f700dda5f7a8cdfc1780dc7af73a49
+size 11717438

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/summary.json

@@ -0,0 +1,6 @@
+{
+    "exact match": 29523,
+    "semantically correct": 58346,
+    "incorrect": 12131,
+    "correct": 87869
+}

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/summary.txt

@@ -0,0 +1,8 @@
+Command Line Arguments:
+--model-path=models-prop/10ap/0000-rop-bn0-fn1-ada1-s43 --seed=42 eval-ted --ds-name=ltl-35-10ap --beam-size=3
+
+EVALUATION SUMMARY
+Correct: 87869/100000, 87.869000%
+Exact match: 29523/100000, 29.523000%
+Semantically correct: 58346/100000, 58.346000%
+Incorrect: 12131/100000, 12.131000%

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/summary.json

@@ -0,0 +1,7 @@
+{
+    "exact match": 41493,
+    "semantically correct": 49374,
+    "incorrect": 9132,
+    "invalid": 1,
+    "correct": 90867
+}

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/summary.txt

@@ -0,0 +1,9 @@
+Command Line Arguments:
+--model-path=models-prop/10ap/0000-rop-bn0-fn1-ada1-s43 eval-ted --beam-size=3
+
+EVALUATION SUMMARY
+Correct: 90867/100000, 90.867000%
+Exact match: 41493/100000, 41.493000%
+Semantically correct: 49374/100000, 49.374000%
+Incorrect: 9132/100000, 9.132000%
+Invalid: 1/100000, 0.001000%

models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/runs/Jan30_11-14-48_as01r5b15/events.out.tfevents.1738232090.as01r5b15.163555.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd098aa2760321cfd10e6d4a2d47755cc572b65b170aa2bca67dd13dff15b3fe
+size 44797

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/command-log.txt

@@ -0,0 +1,67 @@
+Number of parameters: 2_903_676
+Arguments:
+{
+    "model_path": "models-prop/5ap/0000-rop-bn0-fn1-ada1-s42",
+    "device": "cuda",
+    "seed": 42,
+    "subparser": "train-ted",
+    "data_dir": "data-prop",
+    "ds_name": "ltl-35",
+    "max_trace_length": -1,
+    "min_aps": null,
+    "max_aps": null,
+    "exact_aps": null,
+    "vocab_aps": null,
+    "val_split": "val",
+    "merged_vocab": true,
+    "merge_tokens": "all",
+    "dynamic_aps": false,
+    "learning_rate": 0.001,
+    "lr_scheduler_type": "cosine",
+    "warmup_steps": 1000,
+    "weight_decay": 0.1,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.95,
+    "max_grad_norm": 1.0,
+    "epochs": 64,
+    "batch_size": 1024,
+    "grad_acc_steps": 1,
+    "logging_steps": 500,
+    "eval_steps": 3000,
+    "train_max_samples": null,
+    "val_max_samples": 10000,
+    "trace_max_samples": 100,
+    "dry": false,
+    "eval": false,
+    "resume": false,
+    "loss_fct": "adacos",
+    "num_heads": 6,
+    "d_embed_enc": 132,
+    "d_embed_dec": null,
+    "d_ff": 512,
+    "ff_activation": "relu",
+    "num_layers": 6,
+    "dropout": 0.1,
+    "layer_norm_eps": 1e-06,
+    "enc_pe": "sinusoid",
+    "dec_pe": "rope",
+    "no_pe_cross_keys": false,
+    "tree_pos_enc": true,
+    "d_ap": 0,
+    "ap_embed": "randn",
+    "embed_base_normalization": "disabled",
+    "embed_ap_normalization": "disabled",
+    "embed_final_normalization": "l2",
+    "feature_normalization": "l2",
+    "embed_scaling": "sqrtd",
+    "shuffle_aps": null,
+    "action": "train",
+    "model_type": "ted",
+    "decoder_only": false
+}
+Using CUDA device: NVIDIA H100
+Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]
+numpy version: 1.26.2
+torch version: 2.1.1+cu121
+transformers version: 4.39.3
+

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/config.json

@@ -0,0 +1,55 @@
+{
+    "vocab": {
+        "aps": [
+            "a",
+            "b",
+            "c",
+            "d",
+            "e"
+        ],
+        "consts": [
+            "0",
+            "1"
+        ],
+        "trace_ops": [],
+        "ltl_ops": [
+            "!",
+            "&",
+            "|",
+            "=",
+            "^"
+        ],
+        "merge_tokens": "all",
+        "dynamic_aps": false,
+        "use_start_token": true,
+        "use_pad_token": true,
+        "use_eos_token": true
+    },
+    "d_embed_enc": 132,
+    "d_embed_dec": 132,
+    "d_ff": 512,
+    "ff_activation": "relu",
+    "dropout": 0.1,
+    "num_heads": 6,
+    "num_layers": 6,
+    "layer_norm_eps": 1e-06,
+    "merged_embedder": {
+        "tie_embeddings": true,
+        "pad_vocab_size_multiple": 8,
+        "d_ap": 0,
+        "ap_embed": "randn",
+        "base_normalization": "disabled",
+        "ap_normalization": "disabled",
+        "final_normalization": "l2",
+        "feature_normalization": "l2",
+        "embed_scaling": "sqrtd",
+        "shuffle_aps": null
+    },
+    "max_encode_length": 1024,
+    "max_decode_length": 1024,
+    "tree_pos_enc": true,
+    "datatype": "float32",
+    "enc_pe": "sinusoid",
+    "dec_pe": "rope",
+    "no_pe_cross_keys": false
+}

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bd6c2548b27305d6dbe17bac2e7c913388bb22e33c371b5e7318395a5c8854e
+size 1041085

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da1.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6a0b2e7b6aa98a075b70b8031f36fa48ffe6b58777437a44f405e192ab608ee
+size 1110745

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f9de0a6bbd108c71c720a4706b4dd1b2877fc2a70183cb28da63cd873d80fa5
+size 11714750

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/summary.json

@@ -0,0 +1,6 @@
+{
+    "semantically correct": 37680,
+    "exact match": 57941,
+    "incorrect": 4379,
+    "correct": 95621
+}

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/summary.txt

@@ -0,0 +1,8 @@
+Command Line Arguments:
+--model-path=models-prop/5ap/0000-rop-bn0-fn1-ada1-s42 eval-ted --split=test --beam-size=3
+
+EVALUATION SUMMARY
+Correct: 95621/100000, 95.621000%
+Exact match: 57941/100000, 57.941000%
+Semantically correct: 37680/100000, 37.680000%
+Incorrect: 4379/100000, 4.379000%

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json

@@ -0,0 +1,6 @@
+{
+    "exact match": 57872,
+    "semantically correct": 37801,
+    "incorrect": 4327,
+    "correct": 95673
+}

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt

@@ -0,0 +1,8 @@
+Command Line Arguments:
+--model-path=models-prop/5ap/0000-rop-bn0-fn1-ada1-s42 eval-ted --beam-size=3
+
+EVALUATION SUMMARY
+Correct: 95673/100000, 95.673000%
+Exact match: 57872/100000, 57.872000%
+Semantically correct: 37801/100000, 37.801000%
+Incorrect: 4327/100000, 4.327000%

models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/runs/Jan28_01-12-19_as05r4b20/events.out.tfevents.1738023140.as05r4b20.327756.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8088afad1ed66b245a7af4280334a473b80b364e97bc3bc8f480b76b2fb9b4ab
+size 44769

models-prop/5ap/base-rop-bn0-fn1-ada1-s44/command-log.txt

@@ -0,0 +1,67 @@
+Number of parameters: 2_903_676
+Arguments:
+{
+    "model_path": "models-prop/5ap/base-rop-bn0-fn1-ada1-s44",
+    "device": "cuda",
+    "seed": 44,
+    "subparser": "train-ted",
+    "data_dir": "data-prop",
+    "ds_name": "ltl-35-perturbed",
+    "max_trace_length": -1,
+    "min_aps": null,
+    "max_aps": null,
+    "exact_aps": null,
+    "vocab_aps": null,
+    "val_split": "val",
+    "merged_vocab": true,
+    "merge_tokens": "all",
+    "dynamic_aps": false,
+    "learning_rate": 0.001,
+    "lr_scheduler_type": "cosine",
+    "warmup_steps": 1000,
+    "weight_decay": 0.1,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.95,
+    "max_grad_norm": 1.0,
+    "epochs": 64,
+    "batch_size": 1024,
+    "grad_acc_steps": 1,
+    "logging_steps": 500,
+    "eval_steps": 3000,
+    "train_max_samples": null,
+    "val_max_samples": 10000,
+    "trace_max_samples": 100,
+    "dry": false,
+    "eval": false,
+    "resume": false,
+    "loss_fct": "adacos",
+    "num_heads": 6,
+    "d_embed_enc": 132,
+    "d_embed_dec": null,
+    "d_ff": 512,
+    "ff_activation": "relu",
+    "num_layers": 6,
+    "dropout": 0.1,
+    "layer_norm_eps": 1e-06,
+    "enc_pe": "sinusoid",
+    "dec_pe": "rope",
+    "no_pe_cross_keys": false,
+    "tree_pos_enc": true,
+    "d_ap": 0,
+    "ap_embed": "randn",
+    "embed_base_normalization": "disabled",
+    "embed_ap_normalization": "disabled",
+    "embed_final_normalization": "l2",
+    "feature_normalization": "l2",
+    "embed_scaling": "sqrtd",
+    "shuffle_aps": null,
+    "action": "train",
+    "model_type": "ted",
+    "decoder_only": false
+}
+Using CUDA device: NVIDIA H100
+Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]
+numpy version: 1.26.2
+torch version: 2.1.1+cu121
+transformers version: 4.39.3
+

models-prop/5ap/base-rop-bn0-fn1-ada1-s44/config.json

@@ -0,0 +1,55 @@
+{
+    "vocab": {
+        "aps": [
+            "a",
+            "b",
+            "c",
+            "d",
+            "e"
+        ],
+        "consts": [
+            "0",
+            "1"
+        ],
+        "trace_ops": [],
+        "ltl_ops": [
+            "!",
+            "&",
+            "|",
+            "=",
+            "^"
+        ],
+        "merge_tokens": "all",
+        "dynamic_aps": false,
+        "use_start_token": true,
+        "use_pad_token": true,
+        "use_eos_token": true
+    },
+    "d_embed_enc": 132,
+    "d_embed_dec": 132,
+    "d_ff": 512,
+    "ff_activation": "relu",
+    "dropout": 0.1,
+    "num_heads": 6,
+    "num_layers": 6,
+    "layer_norm_eps": 1e-06,
+    "merged_embedder": {
+        "tie_embeddings": true,
+        "pad_vocab_size_multiple": 8,
+        "d_ap": 0,
+        "ap_embed": "randn",
+        "base_normalization": "disabled",
+        "ap_normalization": "disabled",
+        "final_normalization": "l2",
+        "feature_normalization": "l2",
+        "embed_scaling": "sqrtd",
+        "shuffle_aps": null
+    },
+    "max_encode_length": 1024,
+    "max_decode_length": 1024,
+    "tree_pos_enc": true,
+    "datatype": "float32",
+    "enc_pe": "sinusoid",
+    "dec_pe": "rope",
+    "no_pe_cross_keys": false
+}

models-prop/5ap/base-rop-bn0-fn1-ada1-s44/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ede6d600c5e51399f68ddde2ba86c617ba15f9873b4509e54dd399fbb98e3214
+size 11714750

models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/summary.json

@@ -0,0 +1,7 @@
+{
+    "incorrect": 58216,
+    "semantically correct": 32531,
+    "exact match": 9035,
+    "invalid": 218,
+    "correct": 41566
+}

models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt

@@ -0,0 +1,9 @@
+Command Line Arguments:
+--model-path=models-prop/5ap/base-rop-bn0-fn1-ada1-s44 eval-ted --beam-size=3
+
+EVALUATION SUMMARY
+Correct: 41566/100000, 41.566000%
+Exact match: 9035/100000, 9.035000%
+Semantically correct: 32531/100000, 32.531000%
+Incorrect: 58216/100000, 58.216000%
+Invalid: 218/100000, 0.218000%

models-prop/5ap/base-rop-bn0-fn1-ada1-s44/runs/Jan28_01-12-17_as03r3b29/events.out.tfevents.1738023138.as03r3b29.2568593.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c929f62de6a1da0bfbca8f120193478a2bd087579deb16faae4eeb4231729ee
+size 44769

models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/command-log.txt

@@ -0,0 +1,67 @@
+Number of parameters: 2_902_928
+Arguments:
+{
+    "model_path": "models-prop/5ap/d020-rop-bn0-fn1-ada1-s42",
+    "device": "cuda",
+    "seed": 42,
+    "subparser": "train-ted",
+    "data_dir": "data-prop",
+    "ds_name": "ltl-35-perturbed",
+    "max_trace_length": -1,
+    "min_aps": null,
+    "max_aps": null,
+    "exact_aps": null,
+    "vocab_aps": null,
+    "val_split": "val",
+    "merged_vocab": true,
+    "merge_tokens": "all",
+    "dynamic_aps": true,
+    "learning_rate": 0.001,
+    "lr_scheduler_type": "cosine",
+    "warmup_steps": 1000,
+    "weight_decay": 0.1,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.95,
+    "max_grad_norm": 1.0,
+    "epochs": 64,
+    "batch_size": 1024,
+    "grad_acc_steps": 1,
+    "logging_steps": 500,
+    "eval_steps": 3000,
+    "train_max_samples": null,
+    "val_max_samples": 10000,
+    "trace_max_samples": 100,
+    "dry": false,
+    "eval": false,
+    "resume": false,
+    "loss_fct": "adacos",
+    "num_heads": 6,
+    "d_embed_enc": 132,
+    "d_embed_dec": null,
+    "d_ff": 512,
+    "ff_activation": "relu",
+    "num_layers": 6,
+    "dropout": 0.1,
+    "layer_norm_eps": 1e-06,
+    "enc_pe": "sinusoid",
+    "dec_pe": "rope",
+    "no_pe_cross_keys": false,
+    "tree_pos_enc": true,
+    "d_ap": 20,
+    "ap_embed": "diagbor",
+    "embed_base_normalization": "disabled",
+    "embed_ap_normalization": "disabled",
+    "embed_final_normalization": "l2",
+    "feature_normalization": "l2",
+    "embed_scaling": "sqrtd",
+    "shuffle_aps": null,
+    "action": "train",
+    "model_type": "ted",
+    "decoder_only": false
+}
+Using CUDA device: NVIDIA H100
+Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]
+numpy version: 1.26.2
+torch version: 2.1.1+cu121
+transformers version: 4.39.3
+

models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/config.json

@@ -0,0 +1,55 @@
+{
+    "vocab": {
+        "aps": [
+            "a",
+            "b",
+            "c",
+            "d",
+            "e"
+        ],
+        "consts": [
+            "0",
+            "1"
+        ],
+        "trace_ops": [],
+        "ltl_ops": [
+            "!",
+            "&",
+            "|",
+            "=",
+            "^"
+        ],
+        "merge_tokens": "all",
+        "dynamic_aps": true,
+        "use_start_token": true,
+        "use_pad_token": true,
+        "use_eos_token": true
+    },
+    "d_embed_enc": 132,
+    "d_embed_dec": 132,
+    "d_ff": 512,
+    "ff_activation": "relu",
+    "dropout": 0.1,
+    "num_heads": 6,
+    "num_layers": 6,
+    "layer_norm_eps": 1e-06,
+    "merged_embedder": {
+        "tie_embeddings": true,
+        "pad_vocab_size_multiple": 8,
+        "d_ap": 20,
+        "ap_embed": "diagbor",
+        "base_normalization": "disabled",
+        "ap_normalization": "disabled",
+        "final_normalization": "l2",
+        "feature_normalization": "l2",
+        "embed_scaling": "sqrtd",
+        "shuffle_aps": null
+    },
+    "max_encode_length": 1024,
+    "max_decode_length": 1024,
+    "tree_pos_enc": true,
+    "datatype": "float32",
+    "enc_pe": "sinusoid",
+    "dec_pe": "rope",
+    "no_pe_cross_keys": false
+}

models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83c8fc3ec07546be0401ab55376327937250b593ce774b6eadb8471c4b6b0e78
+size 2545486