diff --git a/README.md b/README.md index 7b95401dc46245ac339fc25059d4a56d90b4cde5..61c16f47bcbbcbd5458fdda8d81907b56199ec01 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,65 @@ --- license: apache-2.0 --- + +# Interchangeable Token Embeddings + +This repository contains the models for the ICML 2025 paper, [Interchangeable Token Embeddings for Extendable Vocabulary and Alpha-Equivalence](https://arxiv.org/abs/2410.17161). + +Links: +1. [arXiv](https://arxiv.org/abs/2410.17161) +2. [Project Page](https://necrashter.github.io/interchangeable-token-embeddings/) +3. [Code](https://github.com/necrashter/interchangeable-token-embeddings) + +## Models + +Figure 3a (LTL): +| Model | Path | +| ----- | ---- | +| Proposed Method | `models/ltl-5/d005-rop-bn1-fn1-ada1-s42` | +| Baseline | `models/adacos/nondyn-rope` | +| Full-Vocabulary | `models/10ap/adacos-nondyn` | +| Alpha-Renaming | `models/ltl-5/s010-rop-bn1-fn1-ada1-s46` | + +Figure 3b (Propositional Logic): +| Model | Path | +| ----- | ---- | +| Proposed Method | `models-prop/5ap/d020-rop-bn1-fn1-ada1-s44` | +| Baseline | `models-prop/5ap/0000-rop-bn0-fn1-ada1-s42` | +| Full-Vocabulary | `models-prop/10ap/0000-rop-bn0-fn1-ada1-s43` | +| Alpha-Renaming | `models-prop/5ap/s010-rop-bn1-fn1-ada1-s44` | + +Figure 5a (Ablation - LTL): +| Model | Path | +| ----- | ---- | +| w/o Adacos | `models/ltl-5/d005-rop-bn1-fn1-ada0-s42` | +| w/o $f_{{fn}}$ & AdaCos | `models/ltl-5/d005-rop-bn1-fn0-ada0-s42` | +| w/o $f_{{bn}}$ | `models/ltl-5/d005-rop-bn0-fn1-ada1-s42` | + +Figure 5b (Ablation - Propositional Logic): +| Model | Path | +| ----- | ---- | +| w/o Adacos | `models-prop/5ap/d020-rop-bn1-fn1-ada0-s43` | +| w/o $f_{{fn}}$ & AdaCos | `models-prop/5ap/d020-rop-bn1-fn0-ada0-s44` | +| w/o $f_{{bn}}$ | `models-prop/5ap/d020-rop-bn0-fn1-ada1-s42` | + +### Perturbation Experiments + +Table 2 and 7 (LTL): +| Model | Path | +| ----- | ---- | +| Normal Baseline | `models/adacos/nondyn-rope` | +| Perturbed Baseline | `models/adacos/nondyn-rope-perturbed` | +| Perturbed Alpha-Renaming | `models/adacos/s005-rope-perturbed-s42` | +| Perturbed Proposed | `models/adacos/diagbor4-rope-perturbed` | +| Limited Baseline | `models/adacos/nondyn-rope-limited` | +| Limited Alpha-Renaming | `models/adacos/s005-rope-limited-s42` | +| Limited Proposed | `models/adacos/diagbor4-rope-limited` | + +Table 8 (Propositional Logic): +| Model | Path | +| ----- | ---- | +| Normal Baseline | `models-prop/5ap/0000-rop-bn0-fn1-ada1-s42` | +| Perturbed Baseline | `models-prop/5ap/base-rop-bn0-fn1-ada1-s44` | +| Perturbed Alpha-Renaming | `models-prop/5ap/s005-rop-bn1-fn1-ada1-s43` | +| Perturbed Proposed | `models-prop/5ap/d020-rop-bn1-fn1-ada1-s42` | diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/command-log.txt b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..8b5de4c9eca85da4ba369c4824f0a2eeb5b1b780 --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_904_336 +Arguments: +{ + "model_path": "models-prop/10ap/0000-rop-bn0-fn1-ada1-s43", + "device": "cuda", + "seed": 43, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35-10ap", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": false, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "disabled", + "embed_ap_normalization": "disabled", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/config.json b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/config.json new file mode 100644 index 0000000000000000000000000000000000000000..10acf4129f2e04f257c7e29820f7ec1c23ed483c --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/config.json @@ -0,0 +1,60 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": false, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "disabled", + "ap_normalization": "disabled", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da.pkl b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2c7e407bca25adaad1f829c8bfbb9f08d8e90a0e --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ae22f58ed2164dbda8a13f51c013c5c39e50583c3415e795d5c017144064187 +size 2682581 diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da.png b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..0384ff1ed6eef396bc674f88fa8684fbea92d798 Binary files /dev/null and b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da.png differ diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da1.pkl b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..b1d569b4ad352543979030d1df57b6a7a9e8df08 --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4772a6943f13f92b797149e7e6f206c3ee4c411cf7681fd32fc7be3f669330ae +size 3218204 diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da1.png b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..f57145358ffc06072f8c7ce88e568ef1a359ccb4 Binary files /dev/null and b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/eval2da1.png differ diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/pytorch_model.bin b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..3d5ccfe0eee49c84f302afc242d46a5c1c2ded08 --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91f72633f1aa61db03ec9e1aeeee6e2179f700dda5f7a8cdfc1780dc7af73a49 +size 11717438 diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/size_hist.png b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..bbc455f89184d971e1452d7bf2cbfab3a4d5bf8f Binary files /dev/null and b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/size_hist.svg b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..e0829c3148ee0e47cc0080475111ff69b5741720 --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-30T15:04:31.288141 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/summary.json b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..cb98cf67c9a10f805b77e50004be8b6b0e7cd488 --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 29523, + "semantically correct": 58346, + "incorrect": 12131, + "correct": 87869 +} \ No newline at end of file diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/summary.txt b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..78aa359b51f26eb97a15c28aec9ea527624c70cf --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/10ap/0000-rop-bn0-fn1-ada1-s43 --seed=42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 87869/100000, 87.869000% +Exact match: 29523/100000, 29.523000% +Semantically correct: 58346/100000, 58.346000% +Incorrect: 12131/100000, 12.131000% diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/trace_times.png b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..80adfc561a27d7dc1dd586de0d54f93873142d8e Binary files /dev/null and b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.png b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..99267b8557750b2be03477a34b688039dd116332 Binary files /dev/null and b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.svg b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..3130d491a80fb338802e60a1e975198ad916542b --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1316 @@ + + + + + + + + 2025-01-30T12:54:57.412185 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/summary.json b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..467f8b2e79e5ac393fb9289ae9380732a9e96ea8 --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/summary.json @@ -0,0 +1,7 @@ +{ + "exact match": 41493, + "semantically correct": 49374, + "incorrect": 9132, + "invalid": 1, + "correct": 90867 +} \ No newline at end of file diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/summary.txt b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..e01a005e1f6616ae2e6de5362e5e38105cad9f40 --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,9 @@ +Command Line Arguments: +--model-path=models-prop/10ap/0000-rop-bn0-fn1-ada1-s43 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 90867/100000, 90.867000% +Exact match: 41493/100000, 41.493000% +Semantically correct: 49374/100000, 49.374000% +Incorrect: 9132/100000, 9.132000% +Invalid: 1/100000, 0.001000% diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/trace_times.png b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..190e6f165fad796bb15ca0296bd5f746f561c5e6 Binary files /dev/null and b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/runs/Jan30_11-14-48_as01r5b15/events.out.tfevents.1738232090.as01r5b15.163555.0 b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/runs/Jan30_11-14-48_as01r5b15/events.out.tfevents.1738232090.as01r5b15.163555.0 new file mode 100644 index 0000000000000000000000000000000000000000..cb57e22c363a7da45706d1e4628a671e6ce4a839 --- /dev/null +++ b/models-prop/10ap/0000-rop-bn0-fn1-ada1-s43/runs/Jan30_11-14-48_as01r5b15/events.out.tfevents.1738232090.as01r5b15.163555.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd098aa2760321cfd10e6d4a2d47755cc572b65b170aa2bca67dd13dff15b3fe +size 44797 diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/command-log.txt b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..eccd0243016063c999e7ddab0b4970968a44e18f --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_903_676 +Arguments: +{ + "model_path": "models-prop/5ap/0000-rop-bn0-fn1-ada1-s42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": false, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "disabled", + "embed_ap_normalization": "disabled", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/config.json b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..856163d48a475e303423715008609a407f28639f --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/config.json @@ -0,0 +1,55 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": false, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "disabled", + "ap_normalization": "disabled", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da.pkl b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..38971e4201921448175b113e92499bad0ed32784 --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bd6c2548b27305d6dbe17bac2e7c913388bb22e33c371b5e7318395a5c8854e +size 1041085 diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da.png b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..83fe08fae43a92f2388260c3ba97db914ec3a13c Binary files /dev/null and b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da.png differ diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da1.pkl b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..bf51f50090dce58b08f5e1c7dcb10851a9cd6ea2 --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6a0b2e7b6aa98a075b70b8031f36fa48ffe6b58777437a44f405e192ab608ee +size 1110745 diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da1.png b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..194047b1cdd2e176db18037acb0f94003ed40fa0 Binary files /dev/null and b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/eval2da1.png differ diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/pytorch_model.bin b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..fccecbbe4614614bbb52cb5357f4be6f169fee1c --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f9de0a6bbd108c71c720a4706b4dd1b2877fc2a70183cb28da63cd873d80fa5 +size 11714750 diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.png b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..077323f2e09ccd2267e118abd50fe3da85fc6381 Binary files /dev/null and b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.png differ diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.svg b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..ec2c8655d64d8381ce6e676621dcdf46d29f15a0 --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-30T13:09:15.418685 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/summary.json b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b8b7625cbc171e8df3d6e9db2e5c8b2351ab5a78 --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/summary.json @@ -0,0 +1,6 @@ +{ + "semantically correct": 37680, + "exact match": 57941, + "incorrect": 4379, + "correct": 95621 +} \ No newline at end of file diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/summary.txt b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..a6c9481c1b923e481e2eedab952b342f70d22744 --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/0000-rop-bn0-fn1-ada1-s42 eval-ted --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 95621/100000, 95.621000% +Exact match: 57941/100000, 57.941000% +Semantically correct: 37680/100000, 37.680000% +Incorrect: 4379/100000, 4.379000% diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/trace_times.png b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..a98129ae834385fb33b261ca54685efe6c6581a4 Binary files /dev/null and b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-test-b3/trace_times.png differ diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..33f020b28015cf0863575e798388fa9e8aaa1260 Binary files /dev/null and b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..9464c3f9fd3e15732bc2435dac6905374917fcc5 --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-28T02:49:30.853804 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..844bf4358c563bb0a85aabf0ebab6b2722eba24b --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 57872, + "semantically correct": 37801, + "incorrect": 4327, + "correct": 95673 +} \ No newline at end of file diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..0ef50d59d3bbd7e5233b559fb5394cd77264c40b --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/0000-rop-bn0-fn1-ada1-s42 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 95673/100000, 95.673000% +Exact match: 57872/100000, 57.872000% +Semantically correct: 37801/100000, 37.801000% +Incorrect: 4327/100000, 4.327000% diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..569dafedb1919cbe4e4f1d3868763a7c6894625f Binary files /dev/null and b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/runs/Jan28_01-12-19_as05r4b20/events.out.tfevents.1738023140.as05r4b20.327756.0 b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/runs/Jan28_01-12-19_as05r4b20/events.out.tfevents.1738023140.as05r4b20.327756.0 new file mode 100644 index 0000000000000000000000000000000000000000..cdd8b82ca24a822dbe31bfda80e3f18f27e19774 --- /dev/null +++ b/models-prop/5ap/0000-rop-bn0-fn1-ada1-s42/runs/Jan28_01-12-19_as05r4b20/events.out.tfevents.1738023140.as05r4b20.327756.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8088afad1ed66b245a7af4280334a473b80b364e97bc3bc8f480b76b2fb9b4ab +size 44769 diff --git a/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/command-log.txt b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..ee80b2bbffa7f7c36667deb2d8b8ab6ae00e4605 --- /dev/null +++ b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_903_676 +Arguments: +{ + "model_path": "models-prop/5ap/base-rop-bn0-fn1-ada1-s44", + "device": "cuda", + "seed": 44, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": false, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "disabled", + "embed_ap_normalization": "disabled", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/config.json b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/config.json new file mode 100644 index 0000000000000000000000000000000000000000..856163d48a475e303423715008609a407f28639f --- /dev/null +++ b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/config.json @@ -0,0 +1,55 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": false, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "disabled", + "ap_normalization": "disabled", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/pytorch_model.bin b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..54de2c36ddbcf43dd908b03d20b4d5e25b350cf5 --- /dev/null +++ b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ede6d600c5e51399f68ddde2ba86c617ba15f9873b4509e54dd399fbb98e3214 +size 11714750 diff --git a/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.png b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..b5025e1ac550ec17cc992fe582f33af464b69421 Binary files /dev/null and b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.svg b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..e11250eb6757beaafd86ac89d9659ae77ceea3a0 --- /dev/null +++ b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1316 @@ + + + + + + + + 2025-01-28T02:50:06.476084 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/summary.json b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5461ea4d5f75cc2a114e9070003651449ef752d6 --- /dev/null +++ b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/summary.json @@ -0,0 +1,7 @@ +{ + "incorrect": 58216, + "semantically correct": 32531, + "exact match": 9035, + "invalid": 218, + "correct": 41566 +} \ No newline at end of file diff --git a/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..366854361785e181058e349ee89484b22d665cb0 --- /dev/null +++ b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,9 @@ +Command Line Arguments: +--model-path=models-prop/5ap/base-rop-bn0-fn1-ada1-s44 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 41566/100000, 41.566000% +Exact match: 9035/100000, 9.035000% +Semantically correct: 32531/100000, 32.531000% +Incorrect: 58216/100000, 58.216000% +Invalid: 218/100000, 0.218000% diff --git a/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/trace_times.png b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..31bde7ac7ded4b18b6939294000ff5b8431bd1df Binary files /dev/null and b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/runs/Jan28_01-12-17_as03r3b29/events.out.tfevents.1738023138.as03r3b29.2568593.0 b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/runs/Jan28_01-12-17_as03r3b29/events.out.tfevents.1738023138.as03r3b29.2568593.0 new file mode 100644 index 0000000000000000000000000000000000000000..b47e59b36421323254e4bac6dfededa720e98aba --- /dev/null +++ b/models-prop/5ap/base-rop-bn0-fn1-ada1-s44/runs/Jan28_01-12-17_as03r3b29/events.out.tfevents.1738023138.as03r3b29.2568593.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c929f62de6a1da0bfbca8f120193478a2bd087579deb16faae4eeb4231729ee +size 44769 diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/command-log.txt b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..172a3810117d9eeec826a6b8a24cc1143b691ef5 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_902_928 +Arguments: +{ + "model_path": "models-prop/5ap/d020-rop-bn0-fn1-ada1-s42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 20, + "ap_embed": "diagbor", + "embed_base_normalization": "disabled", + "embed_ap_normalization": "disabled", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/config.json b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..999e5e65c0be2f3fa64367f472435488339d9966 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/config.json @@ -0,0 +1,55 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 20, + "ap_embed": "diagbor", + "base_normalization": "disabled", + "ap_normalization": "disabled", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da.pkl b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..b4c9b3bfb0e5f4d22d76f5d4024ee23e76f20697 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83c8fc3ec07546be0401ab55376327937250b593ce774b6eadb8471c4b6b0e78 +size 2545486 diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da.png b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..e552e1f87ef5ee97191202e4da9ede684a044bbe Binary files /dev/null and b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da.png differ diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da1.pkl b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..b6cfd68373f7f63172670f6d75520d7b9e19e1f2 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1271ecb453eefe163b3a0d1daeecb0506d259df7c2e1c3ee590858ef3f089937 +size 3057990 diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da1.png b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..f1ee4d7b1e3c502d1b62ccdb38c940a10c2e03a0 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/eval2da1.png differ diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/pytorch_model.bin b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..4efe147cbbc234304ce18bd8ded81162687c3531 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae0eb54911acf96bc465eae46f5ee39918dba2ded7ff8e00ecbb217e36eb5aee +size 11711806 diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..a5bc8cbb561422d18b8f7796fd5091750d472592 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..ead8aa36658e3c3fa1c72a5db91fa7b81eea7c4d --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1316 @@ + + + + + + + + 2025-03-30T21:21:01.385153 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..02b23e51979f4a111408eb303c8d27bae26a3392 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,7 @@ +{ + "incorrect": 95920, + "semantically correct": 3097, + "exact match": 982, + "invalid": 1, + "correct": 4079 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..47280970056cc8491ca2524fdf62e0cbc03621ae --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,9 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn0-fn1-ada1-s42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 4079/100000, 4.079000% +Exact match: 982/100000, 0.982000% +Semantically correct: 3097/100000, 3.097000% +Incorrect: 95920/100000, 95.920000% +Invalid: 1/100000, 0.001000% diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..a826727212b2bebe70e6054b6b83b62937edd25c Binary files /dev/null and b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..e139f653ee632ade873014edc32a81696905b9f1 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..7160e2e0f78abb2549efb4e423b55b5e7f96720f --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-03-30T21:14:13.654269 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c198784e467a5717387df7550df642155a7e7a74 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 4584, + "incorrect": 86933, + "semantically correct": 8483, + "correct": 13067 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..52262376d167a131f404de571e4911473ec1004a --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn0-fn1-ada1-s42 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 13067/100000, 13.067000% +Exact match: 4584/100000, 4.584000% +Semantically correct: 8483/100000, 8.483000% +Incorrect: 86933/100000, 86.933000% diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..10b61bb57c65f91732ca940f41d5e74b619f50f2 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/runs/Mar30_19-37-33_as01r1b24/events.out.tfevents.1743356255.as01r1b24.1965030.0 b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/runs/Mar30_19-37-33_as01r1b24/events.out.tfevents.1743356255.as01r1b24.1965030.0 new file mode 100644 index 0000000000000000000000000000000000000000..0f5ec3ebabe204143a089f8308eef260df9e834c --- /dev/null +++ b/models-prop/5ap/d020-rop-bn0-fn1-ada1-s42/runs/Mar30_19-37-33_as01r1b24/events.out.tfevents.1743356255.as01r1b24.1965030.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddb61b4ca7bb9268b620bde73db9b73c9eed892928884904f0a32c5a92adcb37 +size 44771 diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/command-log.txt b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..0e90f95bb7c8fe3e1745561923285ded6f1af4b9 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_902_928 +Arguments: +{ + "model_path": "models-prop/5ap/d020-rop-bn1-fn0-ada0-s44", + "device": "cuda", + "seed": 44, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": null, + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 20, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "disabled", + "feature_normalization": "disabled", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/config.json b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a7238f121f27f2eb4db09338ce0ede4713724479 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/config.json @@ -0,0 +1,55 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 20, + "ap_embed": "diagbor", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "disabled", + "feature_normalization": "disabled", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da.pkl b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6eadd32f92a30c6bae7f2bb77304f1b7ae9e875c --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:573169f658b47ee7b10751513e2033d1521e4f9d06b1dd4b80a258b3b5a813a7 +size 2657292 diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da.png b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..6c8e0023697fb0f030da873dec935e9aa08f4aa6 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da1.pkl b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..84c6fb5028482367d5c2cd1b4ab5d1dcbcb3e28c --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42a289d336f72253e766a6590fccd405a9f7b011c8258223e3cb16197eba33c9 +size 3183824 diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da1.png b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..49c99d622a4b577f66eb4103edc21ee32a9e70f2 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/eval2da1.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/pytorch_model.bin b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..60e5c2df9e960e5f49a9fe9932a21be8874a6714 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c01a2226290c8de83c5d18946753b4995157ca5904adcf0e1a0279227e24bc9 +size 11711806 diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..67e8536621f0dc0df593a9610f0f8b1a01fda56d Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..1e55d9db7fc95ba0412f3f9cdb8cb43ed45a41e9 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-03-30T21:15:43.271780 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ffe8545080c1b9107e2a7770a20c7f692171b82e --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 27062, + "semantically correct": 50145, + "incorrect": 22793, + "correct": 77207 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..42c8efe8f7788f5bd6138c7bb211c06d09e0f520 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn0-ada0-s44 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 77207/100000, 77.207000% +Exact match: 27062/100000, 27.062000% +Semantically correct: 50145/100000, 50.145000% +Incorrect: 22793/100000, 22.793000% diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..7f5b772c8b13eae7ed9122831850be4357cbfc2e Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..f1b0ae64a41d629a2fc3d1c40c7c7bd79dfd8e91 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..8885f8e4a18c4b4bf46c3c5e26fd273c047a849f --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-03-30T21:11:56.736942 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..66a410297a062d2ea0bc49fdf58476f00545274a --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 53555, + "semantically correct": 36560, + "incorrect": 9885, + "correct": 90115 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..fc2bbb5e3f4bbdea2ab38df4ce369e5bbf37942f --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn0-ada0-s44 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 90115/100000, 90.115000% +Exact match: 53555/100000, 53.555000% +Semantically correct: 36560/100000, 36.560000% +Incorrect: 9885/100000, 9.885000% diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..50105b94a8de0780f59be53d01d21bc339cf8745 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/runs/Mar30_19-37-34_as04r5b12/events.out.tfevents.1743356255.as04r5b12.1874114.0 b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/runs/Mar30_19-37-34_as04r5b12/events.out.tfevents.1743356255.as04r5b12.1874114.0 new file mode 100644 index 0000000000000000000000000000000000000000..974a79ad4ad7018f5a8135d10f8619a283d94edc --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn0-ada0-s44/runs/Mar30_19-37-34_as04r5b12/events.out.tfevents.1743356255.as04r5b12.1874114.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:122946932bfb3060160b87f98ea0d7844e04aad7b72cea3ed182325ab4eb2cd2 +size 38724 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/command-log.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..782959fa8122e651ba69b1439783662b71871409 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_902_928 +Arguments: +{ + "model_path": "models-prop/5ap/d020-rop-bn1-fn1-ada0-s43", + "device": "cuda", + "seed": 43, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": null, + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 20, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "disabled", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/config.json b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4c86c93513a10d92728620769c136b0891a7ab0b --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/config.json @@ -0,0 +1,55 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 20, + "ap_embed": "diagbor", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "disabled", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da.pkl b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..3f8fb7d8bdac4176539a0a936e48ab9bd9859326 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5e6738d370cd0003a839db429e167274f90d91dce25629db2e964cf16c33445 +size 2659966 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da.png b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..98be70926ab075a5f326b333ded202f0ec3b1786 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da1.pkl b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1ad00f22c42a6a3a83858fcc09e43f5c59cba149 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c20b2166e011ecb595f89e83431bfa799fd56d9e5de638c0b2ed5d4c38251e4 +size 3184501 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da1.png b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..bc9f086f501d8c03255dabe339fa5a610f06dc4b Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/eval2da1.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/pytorch_model.bin b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..54c60c95cdf7e91bb4b6e7963ebfb77a83be681e --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98dea09ef1f968431f85946ab3c02d31a78b1c1f79d7e4bddb52c8b154f0e922 +size 11711806 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..b1f671c5e4efd4d23378f096f0bbd98f0ca3f60b Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..15dfd1bef3d8da9bdf520274b40b2ee41feee152 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1316 @@ + + + + + + + + 2025-03-30T21:14:34.446399 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d5dd3347ed51c41b39ecde8a394851ab62ca2e89 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,7 @@ +{ + "exact match": 27240, + "semantically correct": 50537, + "incorrect": 22221, + "invalid": 2, + "correct": 77777 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..5cfdb7fcf534e90e5038e572e6c058d1ed398168 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,9 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn1-ada0-s43 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 77777/100000, 77.777000% +Exact match: 27240/100000, 27.240000% +Semantically correct: 50537/100000, 50.537000% +Incorrect: 22221/100000, 22.221000% +Invalid: 2/100000, 0.002000% diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..da62faf16e57df8bbdd572ae9bfdb01e11dae394 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..007c95db0ace0752b9bfc10d21a2175be1f06ca1 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..dd4353e4987492e5b811320797e0994dce2318e4 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-03-30T21:11:03.979938 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b0657a1ad16631e7b1e5ac20c5e009b15c6ae2ff --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 54568, + "semantically correct": 37662, + "incorrect": 7770, + "correct": 92230 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..634b14b4977a5b6a11ed2f04418db74613c705f3 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn1-ada0-s43 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 92230/100000, 92.230000% +Exact match: 54568/100000, 54.568000% +Semantically correct: 37662/100000, 37.662000% +Incorrect: 7770/100000, 7.770000% diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..1dda5e82cd2f0db6edc1d296e2f86c82a4308acc Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/runs/Mar30_19-37-34_as03r3b03/events.out.tfevents.1743356256.as03r3b03.735104.0 b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/runs/Mar30_19-37-34_as03r3b03/events.out.tfevents.1743356256.as03r3b03.735104.0 new file mode 100644 index 0000000000000000000000000000000000000000..43c19259fa122a1fadad30be2e36e5162567875e --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada0-s43/runs/Mar30_19-37-34_as03r3b03/events.out.tfevents.1743356256.as03r3b03.735104.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06c545f8410ba593090a322ec48774ec668871eae8155a49122d02e75697df9a +size 38718 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/command-log.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4edf28035c89db679da229e123041507184db7d0 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_902_928 +Arguments: +{ + "model_path": "models-prop/5ap/d020-rop-bn1-fn1-ada1-s42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 20, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/config.json b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..867d69b1155293d591d65c7a83644f078a96e56c --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/config.json @@ -0,0 +1,55 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 20, + "ap_embed": "diagbor", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/pytorch_model.bin b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..9c631a160c54f2749869360188f27fdedc59bab6 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e46138576c7be66aa7bcd98d8cf077350f46d7b8a9e9a9c8a17c153fe1a9e0d +size 11711806 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..cd335292ee30bab658304ebd4108deeb9250dd41 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..d862edfc7aaa74b62de7dd82a9794db1db184b01 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/size_hist.svg @@ -0,0 +1,1316 @@ + + + + + + + + 2025-01-29T21:53:26.147774 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..21930d8a4a43d42ae4b067491ce9b0c2e93d4c2a --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/summary.json @@ -0,0 +1,7 @@ +{ + "semantically correct": 44772, + "incorrect": 35818, + "exact match": 19408, + "invalid": 2, + "correct": 64180 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..8ce833edac157ac8f323df980daf1ea69e3e5c48 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/summary.txt @@ -0,0 +1,9 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn1-ada1-s42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 64180/100000, 64.180000% +Exact match: 19408/100000, 19.408000% +Semantically correct: 44772/100000, 44.772000% +Incorrect: 35818/100000, 35.818000% +Invalid: 2/100000, 0.002000% diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..e8ba211508c4a576f95db751a2b2cf9662db97e8 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-old-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..029dd3bb864b66e8949d512cb66fb40f3f7e9dc2 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..784d072ac75230ae00a7c7adb1f57c62c29f5079 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-30T15:04:42.015167 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..91eb79b3ed716dfa6c05a7125394674cd778dbcd --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 27493, + "semantically correct": 48979, + "incorrect": 23528, + "correct": 76472 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..ef4679927e9c0fc7c0ac0aff03d56db75c7723bb --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn1-ada1-s42 --seed=42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 76472/100000, 76.472000% +Exact match: 27493/100000, 27.493000% +Semantically correct: 48979/100000, 48.979000% +Incorrect: 23528/100000, 23.528000% diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..89d41e0485c0cc369712f02b6617d5e06aaaf8aa Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..a1adaa104d19f740ca8d702d1ba14db66778786c Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..d36f2617a027216dc004c18fd16d9d157ca82ac0 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-29T21:46:51.998524 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..dcf4d8874527b082eabd73037e946d377977adab --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 56453, + "semantically correct": 36792, + "incorrect": 6755, + "correct": 93245 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..db6b9d45cbdb54ba19b026ad4a9998874b736575 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn1-ada1-s42 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 93245/100000, 93.245000% +Exact match: 56453/100000, 56.453000% +Semantically correct: 36792/100000, 36.792000% +Incorrect: 6755/100000, 6.755000% diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..432796dac13a76f4b4af701979abc3311c5ee53e Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/runs/Jan29_20-08-51_as02r4b23/events.out.tfevents.1738177732.as02r4b23.140567.0 b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/runs/Jan29_20-08-51_as02r4b23/events.out.tfevents.1738177732.as02r4b23.140567.0 new file mode 100644 index 0000000000000000000000000000000000000000..6879475cae7ea276218a618414bba87a3b8833d1 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s42/runs/Jan29_20-08-51_as02r4b23/events.out.tfevents.1738177732.as02r4b23.140567.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:056e19ddfb54b89aaa400f6c03908e612d2eaac24596707f4eae6d08a6c927c3 +size 44759 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/command-log.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..da5c6a916808e3be5459c41c3a9e5df45d486c7e --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_902_928 +Arguments: +{ + "model_path": "models-prop/5ap/d020-rop-bn1-fn1-ada1-s44", + "device": "cuda", + "seed": 44, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 20, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/config.json b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/config.json new file mode 100644 index 0000000000000000000000000000000000000000..867d69b1155293d591d65c7a83644f078a96e56c --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/config.json @@ -0,0 +1,55 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 20, + "ap_embed": "diagbor", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da.pkl b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..f37de0d4ee7196faacd44b519b8902f6b50373f2 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b64528557326310ca6e4ff353553ac3bda9fc5ac13072e5f3a2f24f1a14ffb8 +size 2652531 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..6689e5014d076813593b10d91283ef2c8acacd00 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da1.pkl b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..05b53dcf3a2518d62a8a2660a6feb09ec3b443b9 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abe4f5495530bb4af83e03415dff04568648732cfec65ee82e277137b147b41a +size 3179238 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da1.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..9b19d9114bb070012745937de9681e9dfbc9690e Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/eval2da1.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/pytorch_model.bin b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..24f29c1a82c7e48ed935bebf02057f6c96ff0fd5 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86846a221c7a06fbfe03d01dbaa51e414e68c946768c0a6bdb617b5d834aa49a +size 11711806 diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..8a3b2bb4513f25182ad33db391952a96c7044c8c Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..15568df237732a5859f710b98b5af4a7bed810ba --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-29T21:52:19.930620 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e71c15dd857bbbf6a6c5714621f3b0ba6addae83 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 19446, + "semantically correct": 45811, + "incorrect": 34743, + "correct": 65257 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..55b818f081712ed42cbeee3a9bf30276a9e191bf --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn1-ada1-s44 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 65257/100000, 65.257000% +Exact match: 19446/100000, 19.446000% +Semantically correct: 45811/100000, 45.811000% +Incorrect: 34743/100000, 34.743000% diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..7e67a76ab1ba8015e662eb1fdef56bc9dbdeeb03 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..0184539b6e8ea6278672873b0be88f296c81cb0c Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..4886ab8c7c6f04793d67d2b7886eea949b7c0bdd --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-30T15:04:35.624577 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c5a70da978886ef9b0503a1fd5598e32e8800fbf --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 27432, + "semantically correct": 49667, + "incorrect": 22901, + "correct": 77099 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..05127ed8dbb28d81fe1883bf396b4c5cc49f3f75 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn1-ada1-s44 --seed=42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 77099/100000, 77.099000% +Exact match: 27432/100000, 27.432000% +Semantically correct: 49667/100000, 49.667000% +Incorrect: 22901/100000, 22.901000% diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..6b9757ed6b1637a33a9c1d6775c4187b0370e6ac Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..a823c74e56f85cb2960d926b786a353e2e319925 Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.svg b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..a5f6fa4a5608c29ef0266708ca63804b9d9dac16 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-29T21:45:50.128703 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.json b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1c73a84f44b9ab79c8a3ac56c9bf5bcf9f0ce4a5 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 54927, + "semantically correct": 36734, + "incorrect": 8339, + "correct": 91661 +} \ No newline at end of file diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..5f1dd2d3514ba64b78ad47080668c2ff34526dc6 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/d020-rop-bn1-fn1-ada1-s44 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 91661/100000, 91.661000% +Exact match: 54927/100000, 54.927000% +Semantically correct: 36734/100000, 36.734000% +Incorrect: 8339/100000, 8.339000% diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/trace_times.png b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..cc88069dc7577ef6abb9173c50b99f5a72f0d64a Binary files /dev/null and b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/runs/Jan29_20-08-49_as04r1b05/events.out.tfevents.1738177730.as04r1b05.97928.0 b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/runs/Jan29_20-08-49_as04r1b05/events.out.tfevents.1738177730.as04r1b05.97928.0 new file mode 100644 index 0000000000000000000000000000000000000000..605364b7e89881e57d54e08dc08132d11a6b2189 --- /dev/null +++ b/models-prop/5ap/d020-rop-bn1-fn1-ada1-s44/runs/Jan29_20-08-49_as04r1b05/events.out.tfevents.1738177730.as04r1b05.97928.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd85966440dbf13453c75e7f631655653fe26c2766c7e0d9a49a027d9cf48d8 +size 44759 diff --git a/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/command-log.txt b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..abd47a1813d4188de41975da7996211a2236a99b --- /dev/null +++ b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_903_676 +Arguments: +{ + "model_path": "models-prop/5ap/s005-rop-bn1-fn1-ada1-s43", + "device": "cuda", + "seed": 43, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 5, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/config.json b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0ed313d90c1931c2c837b26b1811f5fcd6ecf7d0 --- /dev/null +++ b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/config.json @@ -0,0 +1,55 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 5 + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/pytorch_model.bin b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..a97712d20c0958080b74d093d5981e1bdc31ce73 --- /dev/null +++ b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37558cba64ae5544d848eb29429490f9cc283bccd96e4de415e21e8142ca9b80 +size 11715074 diff --git a/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.png b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..b2b2b1b3381bcbdf2a142ed959e2a874a60c3e44 Binary files /dev/null and b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.svg b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..3a8ee537bd4f9dd856d19eb57bd71ff8da97a4dd --- /dev/null +++ b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-30T15:28:53.526063 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/summary.json b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..42925874e69b87e5da0f161348d7c66d1fdd9859 --- /dev/null +++ b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 57242, + "semantically correct": 36612, + "incorrect": 6146, + "correct": 93854 +} \ No newline at end of file diff --git a/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/summary.txt b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..33f9c6dd3cc25d9c4605ea94db48a77236f1ade7 --- /dev/null +++ b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/s005-rop-bn1-fn1-ada1-s43 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 93854/100000, 93.854000% +Exact match: 57242/100000, 57.242000% +Semantically correct: 36612/100000, 36.612000% +Incorrect: 6146/100000, 6.146000% diff --git a/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/trace_times.png b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..ac76fe1060205d9899265be91660f6747da0aea5 Binary files /dev/null and b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/runs/Jan30_13-52-56_as01r4b19/events.out.tfevents.1738241578.as01r4b19.3965006.0 b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/runs/Jan30_13-52-56_as01r4b19/events.out.tfevents.1738241578.as01r4b19.3965006.0 new file mode 100644 index 0000000000000000000000000000000000000000..c8142ef22ddc76e16eab575f5672b76abc9e2df4 --- /dev/null +++ b/models-prop/5ap/s005-rop-bn1-fn1-ada1-s43/runs/Jan30_13-52-56_as01r4b19/events.out.tfevents.1738241578.as01r4b19.3965006.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a46becbfe95c31021bd6f95801e1bd5545869fa449e08c000b9571e739a280c +size 44753 diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/command-log.txt b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..da66100e3f18aa44e0a8c26c7793f89eeef6f611 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 2_904_336 +Arguments: +{ + "model_path": "models-prop/5ap/s010-rop-bn1-fn1-ada1-s44", + "device": "cuda", + "seed": 44, + "subparser": "train-ted", + "data_dir": "data-prop", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 64, + "batch_size": 1024, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 6, + "d_embed_enc": 132, + "d_embed_dec": null, + "d_ff": 512, + "ff_activation": "relu", + "num_layers": 6, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 10, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/config.json b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/config.json new file mode 100644 index 0000000000000000000000000000000000000000..0033609f7c18460390b564e890389f659a7da5c2 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/config.json @@ -0,0 +1,55 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [], + "ltl_ops": [ + "!", + "&", + "|", + "=", + "^" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 132, + "d_embed_dec": 132, + "d_ff": 512, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 6, + "num_layers": 6, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 10 + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da.pkl b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..3e37d69ee9a438f89041cdb56ff6ead70b6e23c4 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2f5c4e4e7a79ef7ce2dc2d390db8236119ce825b2e2e4e95e228e6b7b6ecd8f +size 2644001 diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..bc02a7309fa7f127f231cff8082138e38643cdb4 Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da1.pkl b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..cdf5950d5dd5c8c8bb6b8aecd37fa3142d21eb6a --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:334c3c8cfe7bd433646ad7dc6ba6ba2a30436a0093a5e14e6e4a0965282e6cfa +size 3170947 diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da1.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..3b2a8d40c42967307c7028404893b0a0ca20cd07 Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/eval2da1.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/pytorch_model.bin b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..43f24403dfa59014061f7c9fb458947d230c9725 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c2c155d68e5eabcc2951f44f9dfba746eda39b9117615174c976c84545b47ed +size 11717698 diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..b011e4157e0ff49eeb550972683f127a9861a75a Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.svg b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..b20bcce83ecb579f617568941c9b872698ee4465 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-28T12:16:02.963397 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.json b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a311b45ee9d7afa0621de4a75ae50c88cfb6c102 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 17135, + "incorrect": 42008, + "semantically correct": 40857, + "correct": 57992 +} \ No newline at end of file diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.txt b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..27b6917f7413754d885fcf6cf772525dfd8090bd --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/s010-rop-bn1-fn1-ada1-s44 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 57992/100000, 57.992000% +Exact match: 17135/100000, 17.135000% +Semantically correct: 40857/100000, 40.857000% +Incorrect: 42008/100000, 42.008000% diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/trace_times.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..81f695513aae08dc090794e4628a1ddf3a3b725e Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-old-val-b3/trace_times.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..62da8a77840c91810257410a550b3277b0879d74 Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.svg b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..cd72ed4c67d8044697c65efc4767844880167232 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-30T15:05:40.735671 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.json b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8b389531a55278b9ec63ff550ba33c7189149ca7 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 23671, + "semantically correct": 43250, + "incorrect": 33079, + "correct": 66921 +} \ No newline at end of file diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.txt b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..2af431f380e43e6afbc87634b43ec27f775e4625 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/s010-rop-bn1-fn1-ada1-s44 --seed=42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 66921/100000, 66.921000% +Exact match: 23671/100000, 23.671000% +Semantically correct: 43250/100000, 43.250000% +Incorrect: 33079/100000, 33.079000% diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/trace_times.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..c47af8ca86d6a5439903d033116c0481d50012f9 Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/size_hist.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..db45fa9a2b3af264bf190d31779ef459dab7a006 Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/size_hist.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/size_hist.svg b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..61dc18bcf8879460660dc893fa2ee365c3c00e97 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-28T23:27:36.469717 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/summary.json b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8122eb5618321adfa169081a2aedd8942ab4ee7e --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "semantically correct": 4284, + "incorrect": 2304, + "exact match": 3412, + "correct": 7696 +} \ No newline at end of file diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/summary.txt b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..334028be321936445dc68083375e66e9cc79c83e --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/s010-rop-bn1-fn1-ada1-s44 eval-ted --ds-name=ltl-35-5ap-10k --beam-size=3 + +EVALUATION SUMMARY +Correct: 7696/10000, 76.960000% +Exact match: 3412/10000, 34.120000% +Semantically correct: 4284/10000, 42.840000% +Incorrect: 2304/10000, 23.040000% diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/trace_times.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..1ccbce963a46b054ddcf51e5ec5d80de416026de Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-5ap-10k-val-b3/trace_times.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..569c70acf98b1f0f6d948d855f7fbe8ecd2796ec Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.svg b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..d1a6819ba11f2d608b79a95df06c70fe47ffab6e --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1208 @@ + + + + + + + + 2025-01-28T01:49:18.940912 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.json b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..27e5beacf24820957e0686b24433d1cd75533a93 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.json @@ -0,0 +1,6 @@ +{ + "exact match": 45028, + "semantically correct": 34691, + "incorrect": 20281, + "correct": 79719 +} \ No newline at end of file diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..568078b561ddfbc148ee289252b6f3f250cb4a5f --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,8 @@ +Command Line Arguments: +--model-path=models-prop/5ap/s010-rop-bn1-fn1-ada1-s44 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 79719/100000, 79.719000% +Exact match: 45028/100000, 45.028000% +Semantically correct: 34691/100000, 34.691000% +Incorrect: 20281/100000, 20.281000% diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/trace_times.png b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..69609824c8d96ff29cdfa791d1c4920152cfb1c3 Binary files /dev/null and b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/results/ltl-35-val-b3/trace_times.png differ diff --git a/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/runs/Jan28_00-12-48_as01r4b24/events.out.tfevents.1738019569.as01r4b24.1886864.0 b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/runs/Jan28_00-12-48_as01r4b24/events.out.tfevents.1738019569.as01r4b24.1886864.0 new file mode 100644 index 0000000000000000000000000000000000000000..cc64f465d71be3d4af38f8075eb27c5e3f80da39 --- /dev/null +++ b/models-prop/5ap/s010-rop-bn1-fn1-ada1-s44/runs/Jan28_00-12-48_as01r4b24/events.out.tfevents.1738019569.as01r4b24.1886864.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:019c5d773a88124ac73fd955e55a01b1de061308d3725453058f7d5ba41380cf +size 44754 diff --git a/models/10ap/adacos-nondyn/command-log.txt b/models/10ap/adacos-nondyn/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..294d7964155f9d2230d56584c6a36b916447e15a --- /dev/null +++ b/models/10ap/adacos-nondyn/command-log.txt @@ -0,0 +1,65 @@ +Number of parameters: 5_811_072 +Arguments: +{ + "model_path": "models/10ap/adacos-nondyn", + "device": "cuda", + "seed": null, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-10ap", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": false, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/10ap/adacos-nondyn/config.json b/models/10ap/adacos-nondyn/config.json new file mode 100644 index 0000000000000000000000000000000000000000..147125a2e852ca9d94491d2acfc21dc3aa83cc6d --- /dev/null +++ b/models/10ap/adacos-nondyn/config.json @@ -0,0 +1,63 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": false, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd" + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/10ap/adacos-nondyn/eval2da.pkl b/models/10ap/adacos-nondyn/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..bc1ed5e53c7ec5ec289c51b19a188a75f99aa8a1 --- /dev/null +++ b/models/10ap/adacos-nondyn/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33e3c0f66077fa2d34e2f402fffe110f18ab65dfaaff8a3c0d830b2b11bcce7c +size 3017947 diff --git a/models/10ap/adacos-nondyn/eval2da.png b/models/10ap/adacos-nondyn/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..ccb1f9ff9d010c2016a519204909eac32608c3d4 Binary files /dev/null and b/models/10ap/adacos-nondyn/eval2da.png differ diff --git a/models/10ap/adacos-nondyn/eval2da1.pkl b/models/10ap/adacos-nondyn/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..100ba31bbda222504398df2ee36fc5821228ebd0 --- /dev/null +++ b/models/10ap/adacos-nondyn/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:099923bbc2d13b20d1408ac7439df14c7bacd73c1730144ef92bccc97ba6f020 +size 3682219 diff --git a/models/10ap/adacos-nondyn/eval2da1.png b/models/10ap/adacos-nondyn/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..0c5b4f34562be2103acf3c61078d206958fbee1c Binary files /dev/null and b/models/10ap/adacos-nondyn/eval2da1.png differ diff --git a/models/10ap/adacos-nondyn/pytorch_model.bin b/models/10ap/adacos-nondyn/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..10f43b15b37ca57666d65a68b29687d8249c527c --- /dev/null +++ b/models/10ap/adacos-nondyn/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3dc65838e04f0f69fb64d2e67d57769307be7e9354f0f7051fad2a4d814b915 +size 23380558 diff --git a/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/size_hist.png b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..a6ffa5a1c98d9cf69338974121efe2f778d382a9 Binary files /dev/null and b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/size_hist.svg b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..010a65d112f17f473ae046e8901e17cee5969063 --- /dev/null +++ b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-09-28T05:00:33.097232 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/summary.json b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fc1f249ab953873db339852331c8ace7d7adfa06 --- /dev/null +++ b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 87764, + "semantically correct": 10171, + "invalid": 178, + "incorrect": 1849, + "timeout": 36, + "runtime error": 2, + "correct": 97935 +} \ No newline at end of file diff --git a/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/summary.txt b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..013500890a7e45c5f20faaffd1c96e32fb415c57 --- /dev/null +++ b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/10ap/adacos-nondyn eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 97935/100000, 97.935000% +Exact match: 87764/100000, 87.764000% +Semantically correct: 10171/100000, 10.171000% +Incorrect: 1849/100000, 1.849000% +Invalid: 178/100000, 0.178000% +Timeout: 36/100000, 0.036000% +Runtime error: 2/100000, 0.002000% diff --git a/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/trace_times.png b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..fed374447dd36f1119033993f5ad9a8070477265 Binary files /dev/null and b/models/10ap/adacos-nondyn/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models/10ap/adacos-nondyn/results/ltl-35-test-b3/size_hist.png b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..33f8a2f959bdc581d85f3b7d5e78dd8a3c2d7b4b Binary files /dev/null and b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/10ap/adacos-nondyn/results/ltl-35-test-b3/size_hist.svg b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..f173ed2bc9341fd4b7224ddc4ed4dd92d713af5e --- /dev/null +++ b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-28T10:53:09.234681 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/10ap/adacos-nondyn/results/ltl-35-test-b3/summary.json b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..180c591a2b366d6e3d3c8e109123eee86e88cb36 --- /dev/null +++ b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "semantically correct": 16957, + "exact match": 80410, + "incorrect": 2482, + "invalid": 109, + "timeout": 29, + "runtime error": 2, + "correct": 97367 +} \ No newline at end of file diff --git a/models/10ap/adacos-nondyn/results/ltl-35-test-b3/summary.txt b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..73e53e5589efa453262c6aa8fa5fab8b5da86db1 --- /dev/null +++ b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/10ap/adacos-nondyn eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 97367/99989, 97.377712% +Exact match: 80410/99989, 80.418846% +Semantically correct: 16957/99989, 16.958865% +Incorrect: 2482/99989, 2.482273% +Invalid: 109/99989, 0.109012% +Timeout: 29/99989, 0.029003% +Runtime error: 2/99989, 0.002000% diff --git a/models/10ap/adacos-nondyn/results/ltl-35-test-b3/trace_times.png b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..c3fbad3a597ed2fb573012065748df4e63a27026 Binary files /dev/null and b/models/10ap/adacos-nondyn/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/10ap/adacos-nondyn/results/ltl-35-val-b3/size_hist.png b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..37aa61971f18a607abe67afd71f713074949e91a Binary files /dev/null and b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/10ap/adacos-nondyn/results/ltl-35-val-b3/size_hist.svg b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..01be72be802d1c7dc23c7091b7e746148b693ce2 --- /dev/null +++ b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-09-28T05:06:23.050429 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/10ap/adacos-nondyn/results/ltl-35-val-b3/summary.json b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..845bec71eac070b45272ea35e7d74b7fef6fbdf7 --- /dev/null +++ b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 80240, + "incorrect": 2527, + "semantically correct": 17086, + "invalid": 105, + "timeout": 29, + "runtime error": 2, + "correct": 97326 +} \ No newline at end of file diff --git a/models/10ap/adacos-nondyn/results/ltl-35-val-b3/summary.txt b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..3f503588b122298a8ca22b22a4d766f528e8cd04 --- /dev/null +++ b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/10ap/adacos-nondyn eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 97326/99989, 97.336707% +Exact match: 80240/99989, 80.248827% +Semantically correct: 17086/99989, 17.087880% +Incorrect: 2527/99989, 2.527278% +Invalid: 105/99989, 0.105012% +Timeout: 29/99989, 0.029003% +Runtime error: 2/99989, 0.002000% diff --git a/models/10ap/adacos-nondyn/results/ltl-35-val-b3/trace_times.png b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..6dd3008d5589556e7a9f9201c25f3762d8c24386 Binary files /dev/null and b/models/10ap/adacos-nondyn/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/10ap/adacos-nondyn/runs/Sep28_02-27-04_as01r5b19/events.out.tfevents.1727483225.as01r5b19.1195020.0 b/models/10ap/adacos-nondyn/runs/Sep28_02-27-04_as01r5b19/events.out.tfevents.1727483225.as01r5b19.1195020.0 new file mode 100644 index 0000000000000000000000000000000000000000..ac170eb9346ec47dd96cad1f339af4f23b4b382b --- /dev/null +++ b/models/10ap/adacos-nondyn/runs/Sep28_02-27-04_as01r5b19/events.out.tfevents.1727483225.as01r5b19.1195020.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9f459c5c2654d08c594a2eb1577789a405ae354763b574dacf5ee9b9e994067 +size 46609 diff --git a/models/adacos/diagbor4-rope-limited/command-log.txt b/models/adacos/diagbor4-rope-limited/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a12c788f185041f0a23ad04eca4a4e7c05d21a3e --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/command-log.txt @@ -0,0 +1,65 @@ +Number of parameters: 5_809_864 +Arguments: +{ + "model_path": "models/diagbor4-rope-limited", + "device": "cuda", + "seed": null, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": 80000, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 4, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/adacos/diagbor4-rope-limited/config.json b/models/adacos/diagbor4-rope-limited/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1b0a0854f9fbf587a831b341deec463530326541 --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/config.json @@ -0,0 +1,58 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 4, + "ap_embed": "diagbor", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd" + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-limited/pytorch_model.bin b/models/adacos/diagbor4-rope-limited/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..174883f35cbf0028a4345852d6df88145361dbbe --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2417fe098d10748820988563222160a265ea42e3eb5a391b5089c4c967bb79e4 +size 23375694 diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/size_hist.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..9c4c193f237abfdc491dcc43cc7596e05e410bda Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/size_hist.svg b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..e657044466712dd5884b3bc9b0f78dffb357c19b --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-10-01T20:46:15.231777 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/summary.json b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..79804a155ecce2590aa85a84c0c0385331bcb961 --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 58940, + "incorrect": 16994, + "semantically correct": 23030, + "timeout": 86, + "invalid": 946, + "runtime error": 4, + "correct": 81970 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/summary.txt b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..66cfc7c776d799aa074e3e1ae092ac729382410f --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/diagbor4-rope-limited eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 81970/100000, 81.970000% +Exact match: 58940/100000, 58.940000% +Semantically correct: 23030/100000, 23.030000% +Incorrect: 16994/100000, 16.994000% +Timeout: 86/100000, 0.086000% +Invalid: 946/100000, 0.946000% +Runtime error: 4/100000, 0.004000% diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/trace_times.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..e777497add5522fbd6c72b75fecf09da879995c8 Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..792c2e3d0e7253b5246c46884f1c6387498394c2 Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.svg b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..29ddb21c50db722edcac9cfdeceaa8c01888f5cb --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-26T23:40:47.585490 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.json b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3493cf2aebe483bf15117bf67280ebbe650eeb0e --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 5976, + "semantically correct": 2817, + "incorrect": 1138, + "invalid": 65, + "timeout": 3, + "runtime error": 1, + "correct": 8793 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.txt b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..3eb74786dd5471a6d119cfdcfbfcc9a6f14f6bdc --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/diagbor4-rope-limited eval-ted --ds-name=ltl-35-perturbed --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 8793/10000, 87.930000% +Exact match: 5976/10000, 59.760000% +Semantically correct: 2817/10000, 28.170000% +Incorrect: 1138/10000, 11.380000% +Invalid: 65/10000, 0.650000% +Timeout: 3/10000, 0.030000% +Runtime error: 1/10000, 0.010000% diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/trace_times.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..e00a4b05a0247057b158fa7fc38da755b6938b48 Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-perturbed-val10k-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/size_hist.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..e4998e23330df72c25d893036d3c49456a825fbf Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/size_hist.svg b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..fa5c14f3837ed1b99760e137a7ec116e452e09bd --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-21T19:38:53.192116 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/summary.json b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..599555d2d3d4ffe707aec71c3c44522425e880de --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "semantically correct": 28271, + "exact match": 59041, + "incorrect": 12262, + "invalid": 369, + "timeout": 41, + "runtime error": 5, + "correct": 87312 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/summary.txt b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..16957a978bb0617f431e72b1c1a137c47f4e2328 --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/diagbor4-rope-limited eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 87312/99989, 87.321605% +Exact match: 59041/99989, 59.047495% +Semantically correct: 28271/99989, 28.274110% +Incorrect: 12262/99989, 12.263349% +Invalid: 369/99989, 0.369041% +Timeout: 41/99989, 0.041005% +Runtime error: 5/99989, 0.005001% diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/trace_times.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..0892e6d8c3032102eb0cc046448324917a6a98e9 Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/size_hist.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..d3b9317b5b6a09f97ba682577d178b86b7ef4671 Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/size_hist.svg b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..2225c974ee24fb3ca5446b3b89ed4a1ee9783c33 --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-09-27T12:15:03.573893 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/summary.json b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8be8791e293606359332592b50868ede9db8bfa8 --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 59096, + "semantically correct": 28304, + "incorrect": 12189, + "invalid": 360, + "timeout": 38, + "runtime error": 2, + "correct": 87400 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/summary.txt b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6ea941398da9f5bbf2d51f1340825f740fa2ceb --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/diagbor4-rope-limited eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 87400/99989, 87.409615% +Exact match: 59096/99989, 59.102501% +Semantically correct: 28304/99989, 28.307114% +Incorrect: 12189/99989, 12.190341% +Invalid: 360/99989, 0.360040% +Timeout: 38/99989, 0.038004% +Runtime error: 2/99989, 0.002000% diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/trace_times.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..9cd1fea36c0d1a53307a259280ed12f544b2e416 Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/size_hist.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..4662e223c6694be42c18b5515a77ba95e328cc7e Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/size_hist.svg b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..9043b1459adf6157c2ff0562d7bc3b09313f3363 --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-26T23:41:56.120381 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/summary.json b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5a94984e46c1a57b935fe01057ea734ed7f69c02 --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 5981, + "semantically correct": 2781, + "incorrect": 1182, + "invalid": 52, + "timeout": 3, + "runtime error": 1, + "correct": 8762 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/summary.txt b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..8fc58f768f67c5ff86d16078e6a109d8a1977a2e --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/diagbor4-rope-limited eval-ted --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 8762/10000, 87.620000% +Exact match: 5981/10000, 59.810000% +Semantically correct: 2781/10000, 27.810000% +Incorrect: 1182/10000, 11.820000% +Invalid: 52/10000, 0.520000% +Timeout: 3/10000, 0.030000% +Runtime error: 1/10000, 0.010000% diff --git a/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/trace_times.png b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..e00a4b05a0247057b158fa7fc38da755b6938b48 Binary files /dev/null and b/models/adacos/diagbor4-rope-limited/results/ltl-35-val10k-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-limited/runs/Sep26_23-26-49_as01r1b04/events.out.tfevents.1727386009.as01r1b04.1110897.0 b/models/adacos/diagbor4-rope-limited/runs/Sep26_23-26-49_as01r1b04/events.out.tfevents.1727386009.as01r1b04.1110897.0 new file mode 100644 index 0000000000000000000000000000000000000000..cc4bfa7934b18c54a978f7aee5818c0a63eddd91 --- /dev/null +++ b/models/adacos/diagbor4-rope-limited/runs/Sep26_23-26-49_as01r1b04/events.out.tfevents.1727386009.as01r1b04.1110897.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c078a8a62f39edb25c04c56e8e72030c39266c2a07d0b196033027e0b6b5272e +size 8631 diff --git a/models/adacos/diagbor4-rope-perturbed/command-log.txt b/models/adacos/diagbor4-rope-perturbed/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..530bd402460c0f8b5be99bde763e3c86ce9324ce --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/command-log.txt @@ -0,0 +1,64 @@ +Number of parameters: 5_809_864 +Arguments: +{ + "model_path": "models/diagbor4-rope-perturbed", + "device": "cuda", + "seed": null, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "tree_pos_enc": true, + "d_ap": 4, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/adacos/diagbor4-rope-perturbed/config.json b/models/adacos/diagbor4-rope-perturbed/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f0bfacc1f1688a5dfbc826100af8b501c9b3ab19 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/config.json @@ -0,0 +1,57 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 4, + "ap_embed": "diagbor", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd" + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope" +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-perturbed/pytorch_model.bin b/models/adacos/diagbor4-rope-perturbed/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..d914d2e0f2e81fb1e8c7acfb8e722d1588183999 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dca916d89384744be120dc884154649c8d82b7da9b1870fd3e927d739f914fb +size 23375694 diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/size_hist.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..10b59adf54cf828592f1afb52bd7cea606391a22 Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/size_hist.svg b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..18c44ffb57d549da9415d0fe743c293c3f2c312b --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-10-01T20:46:19.204126 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/summary.json b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e6f2399951106f3db36fb0de02df9764e604e1c8 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 68230, + "semantically correct": 19522, + "incorrect": 11095, + "invalid": 1083, + "timeout": 65, + "runtime error": 5, + "correct": 87752 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/summary.txt b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..34f1a29075d7e14ac78b32409e9a62bcb43b1240 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/diagbor4-rope-perturbed eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 87752/100000, 87.752000% +Exact match: 68230/100000, 68.230000% +Semantically correct: 19522/100000, 19.522000% +Incorrect: 11095/100000, 11.095000% +Invalid: 1083/100000, 1.083000% +Timeout: 65/100000, 0.065000% +Runtime error: 5/100000, 0.005000% diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/trace_times.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..54801fbc20d43708d226c1b83db53c2353744619 Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..ab3de1445251ed307c4ff6a53e1213c181c21d31 Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.svg b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..5bd5b6dbe99727debff3b472e2fc79a7a50fbd4c --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-25T22:50:52.098870 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.json b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ce59606ba3bafef07906a906b615c68d248ddc9b --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 7582, + "incorrect": 500, + "semantically correct": 1891, + "invalid": 24, + "timeout": 2, + "runtime error": 1, + "correct": 9473 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.txt b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..0f53efb047666f846db55d1e2bdaaf12946ae638 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/diagbor4-rope-perturbed eval-ted --ds-name=ltl-35-perturbed --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 9473/10000, 94.730000% +Exact match: 7582/10000, 75.820000% +Semantically correct: 1891/10000, 18.910000% +Incorrect: 500/10000, 5.000000% +Invalid: 24/10000, 0.240000% +Timeout: 2/10000, 0.020000% +Runtime error: 1/10000, 0.010000% diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/trace_times.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..60ca5e5d3f05c43ce43d124a75f15150b306bafc Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-perturbed-val10k-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/size_hist.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..ff7ba322b99d4c988bd82d3743bcc3d99e086e14 Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/size_hist.svg b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..e983ef3db2f6cf9265b816c9596226ae82675aea --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-21T20:46:27.487205 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/summary.json b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..46a13c74ab88300985f966a83ac3e30d7d2649d0 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 76443, + "semantically correct": 19491, + "incorrect": 3711, + "invalid": 315, + "timeout": 25, + "runtime error": 4, + "correct": 95934 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/summary.txt b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..bd87032957cf2e9b838c175da0c702c7a37f1b1a --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/diagbor4-rope-perturbed eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 95934/99989, 95.944554% +Exact match: 76443/99989, 76.451410% +Semantically correct: 19491/99989, 19.493144% +Incorrect: 3711/99989, 3.711408% +Invalid: 315/99989, 0.315035% +Timeout: 25/99989, 0.025003% +Runtime error: 4/99989, 0.004000% diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/trace_times.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..2f68a7c68e3e73168a818ebb37e7d804c4ffea21 Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/size_hist.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..794765eb80a03aa450cfa1ea605c0a8b41a79ec3 Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/size_hist.svg b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..f4c57e2a7c8cd2de77090d073a4c1d0914f0b126 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-09-27T11:44:15.233092 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/summary.json b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ceb8c0a9ac7e9693ab2e817ab364d2fbd112ef65 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 76219, + "semantically correct": 20064, + "incorrect": 3336, + "invalid": 348, + "timeout": 20, + "runtime error": 2, + "correct": 96283 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/summary.txt b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9015963e7520cb68a82d9f64995ab6525f20a16 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/diagbor4-rope-perturbed eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 96283/99989, 96.293592% +Exact match: 76219/99989, 76.227385% +Semantically correct: 20064/99989, 20.066207% +Incorrect: 3336/99989, 3.336367% +Invalid: 348/99989, 0.348038% +Timeout: 20/99989, 0.020002% +Runtime error: 2/99989, 0.002000% diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/trace_times.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..d0d6fa6fe4c6a3ee2d0f90b357c264d745cf3ee5 Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/size_hist.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..7931ed774cc0fa78a8c990787f9fe4ec52f60afd Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/size_hist.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/size_hist.svg b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..1283c4f11b72cad7cb6fcbc5a18d140d171cffe6 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-25T22:51:58.996500 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/summary.json b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e60b533ed8c5566404deff77406bcb95cbaefe62 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 7678, + "semantically correct": 1907, + "incorrect": 383, + "invalid": 29, + "timeout": 2, + "runtime error": 1, + "correct": 9585 +} \ No newline at end of file diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/summary.txt b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..782dae584c2b7231f5801c8e1f74af99e89b9c54 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/diagbor4-rope-perturbed eval-ted --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 9585/10000, 95.850000% +Exact match: 7678/10000, 76.780000% +Semantically correct: 1907/10000, 19.070000% +Incorrect: 383/10000, 3.830000% +Invalid: 29/10000, 0.290000% +Timeout: 2/10000, 0.020000% +Runtime error: 1/10000, 0.010000% diff --git a/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/trace_times.png b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..60ca5e5d3f05c43ce43d124a75f15150b306bafc Binary files /dev/null and b/models/adacos/diagbor4-rope-perturbed/results/ltl-35-val10k-b3/trace_times.png differ diff --git a/models/adacos/diagbor4-rope-perturbed/runs/Sep25_20-38-27_as02r2b27/events.out.tfevents.1727289508.as02r2b27.1165556.0 b/models/adacos/diagbor4-rope-perturbed/runs/Sep25_20-38-27_as02r2b27/events.out.tfevents.1727289508.as02r2b27.1165556.0 new file mode 100644 index 0000000000000000000000000000000000000000..0b9424a3f8145837ba3ea1c2ec01cb70c11fca81 --- /dev/null +++ b/models/adacos/diagbor4-rope-perturbed/runs/Sep25_20-38-27_as02r2b27/events.out.tfevents.1727289508.as02r2b27.1165556.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6123140f344ab262adab77fd3190349472df136f09b25a754aab504640fc4b6 +size 46573 diff --git a/models/adacos/nondyn-rope-limited/command-log.txt b/models/adacos/nondyn-rope-limited/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..88e7099210ae960f4579785d217375c4efd40059 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/command-log.txt @@ -0,0 +1,65 @@ +Number of parameters: 5_810_432 +Arguments: +{ + "model_path": "models/nondyn-rope-limited", + "device": "cuda", + "seed": null, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": false, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": 80000, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/adacos/nondyn-rope-limited/config.json b/models/adacos/nondyn-rope-limited/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cab1552da51c559bc7480f11904937979f943d23 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/config.json @@ -0,0 +1,58 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": false, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd" + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-limited/pytorch_model.bin b/models/adacos/nondyn-rope-limited/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..bfa14f2ea59e99265f5188a7e9b2620805b8e9e5 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c254260ced9b3948ecfa5e15f881a8427d0c5971f6c02c9bc61abc38e051c4b +size 23377998 diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.png b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..3cd8c433f0b4e95399b6a343cf91aa40899be025 Binary files /dev/null and b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.svg b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..d09fe833924d9e77c705657496cc97052f0a3959 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-26T23:38:51.286101 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.json b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..68535b8d2664a4eba1f7ade4a382692bab4a4872 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 6400, + "semantically correct": 2356, + "incorrect": 1227, + "invalid": 14, + "timeout": 2, + "runtime error": 1, + "correct": 8756 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.txt b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..13d7d9d833e2cc938fad649cb0fb04866a3d2813 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/nondyn-rope-limited eval-ted --ds-name=ltl-35-perturbed --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 8756/10000, 87.560000% +Exact match: 6400/10000, 64.000000% +Semantically correct: 2356/10000, 23.560000% +Incorrect: 1227/10000, 12.270000% +Invalid: 14/10000, 0.140000% +Timeout: 2/10000, 0.020000% +Runtime error: 1/10000, 0.010000% diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/trace_times.png b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..60ca5e5d3f05c43ce43d124a75f15150b306bafc Binary files /dev/null and b/models/adacos/nondyn-rope-limited/results/ltl-35-perturbed-val10k-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/size_hist.png b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..2af0ca582ba1681ecfdc7798d56d7a2a11ea1f95 Binary files /dev/null and b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/size_hist.svg b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..8ea7c4a85ef3f1c4462cebb27e799847835b7a36 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-21T19:37:46.211366 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/summary.json b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..07df8a432e59c234f2ce82192255f995b1a1abba --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "semantically correct": 23855, + "exact match": 63606, + "incorrect": 12367, + "invalid": 116, + "timeout": 41, + "runtime error": 4, + "correct": 87461 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/summary.txt b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..aa74bf3a0242836942fd72f77609d6bbc4af2fa9 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/nondyn-rope-limited eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 87461/99989, 87.470622% +Exact match: 63606/99989, 63.612997% +Semantically correct: 23855/99989, 23.857624% +Incorrect: 12367/99989, 12.368361% +Invalid: 116/99989, 0.116013% +Timeout: 41/99989, 0.041005% +Runtime error: 4/99989, 0.004000% diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/trace_times.png b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..ad2f81bb2fd75e96987f0cfd5a1c0806eb233754 Binary files /dev/null and b/models/adacos/nondyn-rope-limited/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/size_hist.png b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..95688857de9170508d6886bccb0cf378b972bca7 Binary files /dev/null and b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/size_hist.svg b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..412deb2f2116ba7f9c48235515cb81d999b1c9a3 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-09-27T12:21:09.275944 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/summary.json b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..98f7f00f52f811a8cb235e41adb4bbbd4d71e1d3 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 63781, + "semantically correct": 23636, + "incorrect": 12431, + "invalid": 103, + "timeout": 36, + "runtime error": 2, + "correct": 87417 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/summary.txt b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..c879d0e49da7a265a0e470c3ceb6df45832262fb --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/nondyn-rope-limited eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 87417/99989, 87.426617% +Exact match: 63781/99989, 63.788017% +Semantically correct: 23636/99989, 23.638600% +Incorrect: 12431/99989, 12.432368% +Invalid: 103/99989, 0.103011% +Timeout: 36/99989, 0.036004% +Runtime error: 2/99989, 0.002000% diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/trace_times.png b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..ec72546b694cf414d3e47a179510da9868b0d99c Binary files /dev/null and b/models/adacos/nondyn-rope-limited/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/size_hist.png b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..b8a372857849e963bffc304aa58574247578580f Binary files /dev/null and b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/size_hist.svg b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..21dd119da0967fcad10e861600c3f79ccea27484 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-26T23:40:00.078753 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/summary.json b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..821986abfe51b60a1ae123b722b2c53c0cd2e0ad --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 6406, + "semantically correct": 2352, + "incorrect": 1227, + "invalid": 11, + "timeout": 3, + "runtime error": 1, + "correct": 8758 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/summary.txt b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..4c9d3ee4073ccf1036d800b63c0911c04c53ecdd --- /dev/null +++ b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/nondyn-rope-limited eval-ted --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 8758/10000, 87.580000% +Exact match: 6406/10000, 64.060000% +Semantically correct: 2352/10000, 23.520000% +Incorrect: 1227/10000, 12.270000% +Invalid: 11/10000, 0.110000% +Timeout: 3/10000, 0.030000% +Runtime error: 1/10000, 0.010000% diff --git a/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/trace_times.png b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..8fc0a20a308347b8b76eee063b8b5d3bf2a3ad5b Binary files /dev/null and b/models/adacos/nondyn-rope-limited/results/ltl-35-val10k-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope-limited/runs/Sep26_23-24-49_as01r1b04/events.out.tfevents.1727385890.as01r1b04.1110412.0 b/models/adacos/nondyn-rope-limited/runs/Sep26_23-24-49_as01r1b04/events.out.tfevents.1727385890.as01r1b04.1110412.0 new file mode 100644 index 0000000000000000000000000000000000000000..1390785cffc5e40f58acef2a98f25fc16e558cf4 --- /dev/null +++ b/models/adacos/nondyn-rope-limited/runs/Sep26_23-24-49_as01r1b04/events.out.tfevents.1727385890.as01r1b04.1110412.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2693b307a2ffd312d3afeb0bd337eebf8c008b5ba25fec7fd85a71340fa0367e +size 8624 diff --git a/models/adacos/nondyn-rope-perturbed/command-log.txt b/models/adacos/nondyn-rope-perturbed/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..5ec1ff2835828d24f46e0fe251434bb5917fcef6 --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/command-log.txt @@ -0,0 +1,64 @@ +Number of parameters: 5_810_432 +Arguments: +{ + "model_path": "models/nondyn-rope-perturbed", + "device": "cuda", + "seed": null, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": false, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/adacos/nondyn-rope-perturbed/config.json b/models/adacos/nondyn-rope-perturbed/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f14ff6cb6cc9c671f8628e0a6e7de08bf9b9fdae --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/config.json @@ -0,0 +1,57 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": false, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd" + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope" +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-perturbed/pytorch_model.bin b/models/adacos/nondyn-rope-perturbed/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..61a724ea7ef36f3ccb6c8db58f7bc90390bfde55 --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a79753633a9061999f20a0cdc9705130d26e289760891d4c54051b7f1e3cbbdf +size 23377998 diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.png b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..4b57d6f56b0d3f8e6d2a82b7a57dc8fad1b8cc70 Binary files /dev/null and b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.svg b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..f03850b8532aec41efba698d36347d59fbd73a4d --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-25T22:50:46.240528 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.json b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3c9ae2747388d512d124ce50461cb994a0f6367c --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 8372, + "incorrect": 197, + "semantically correct": 1371, + "invalid": 58, + "timeout": 1, + "runtime error": 1, + "correct": 9743 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.txt b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..0f1ad6bbf29a941a1ffb5e09fb6eb6d3b5c1e40e --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/nondyn-rope-perturbed eval-ted --ds-name=ltl-35-perturbed --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 9743/10000, 97.430000% +Exact match: 8372/10000, 83.720000% +Semantically correct: 1371/10000, 13.710000% +Incorrect: 197/10000, 1.970000% +Invalid: 58/10000, 0.580000% +Timeout: 1/10000, 0.010000% +Runtime error: 1/10000, 0.010000% diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/trace_times.png b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..ea7e9e36efdcc5b94e8120056f2929a72f83a9b6 Binary files /dev/null and b/models/adacos/nondyn-rope-perturbed/results/ltl-35-perturbed-val10k-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/size_hist.png b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..dbdaf6bb52f7512b5232f236d5da9bcbe51601bd Binary files /dev/null and b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/size_hist.svg b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..c269d5a81c145a31dddd8e9300b0fbf034031fec --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-21T19:38:10.652321 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/summary.json b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bab561e100e70732efdd43f2f45cf8df369312b0 --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "incorrect": 63653, + "semantically correct": 22009, + "exact match": 12122, + "invalid": 2108, + "timeout": 92, + "runtime error": 5, + "correct": 34131 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/summary.txt b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..d78293206f5aeec4a1d6a9823ad40fec4fdc5544 --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/nondyn-rope-perturbed eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 34131/99989, 34.134755% +Exact match: 12122/99989, 12.123334% +Semantically correct: 22009/99989, 22.011421% +Incorrect: 63653/99989, 63.660003% +Invalid: 2108/99989, 2.108232% +Timeout: 92/99989, 0.092010% +Runtime error: 5/99989, 0.005001% diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/trace_times.png b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..b3e27452aba8c0c1ff524712645344721885de54 Binary files /dev/null and b/models/adacos/nondyn-rope-perturbed/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/size_hist.png b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..7147fad869b61795a14e5d3e3dabf1188583b8a5 Binary files /dev/null and b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/size_hist.svg b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..b3b77464d868e8a940002d7a6dbfb937b90c1e98 --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-09-27T13:40:23.464819 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/summary.json b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e2aefa6cc53ba033f7a4d0e67d8d3d43940a7c2d --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "incorrect": 63770, + "exact match": 11940, + "semantically correct": 22052, + "invalid": 2145, + "timeout": 78, + "runtime error": 4, + "correct": 33992 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/summary.txt b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..1c1380d178d7a391fb3813e86dbc8792c2db0a07 --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/nondyn-rope-perturbed eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 33992/99989, 33.995740% +Exact match: 11940/99989, 11.941314% +Semantically correct: 22052/99989, 22.054426% +Incorrect: 63770/99989, 63.777015% +Invalid: 2145/99989, 2.145236% +Timeout: 78/99989, 0.078009% +Runtime error: 4/99989, 0.004000% diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/trace_times.png b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..3c9caaa2079955d132d904931d3a68b1429c72e3 Binary files /dev/null and b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/size_hist.png b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..22cf856856925fe1030ce94eee4a8a0bb84db70b Binary files /dev/null and b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/size_hist.svg b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..c1ab3a6544290b6a13ff1567893c028ec616f98d --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-25T22:51:55.836524 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/summary.json b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e3301a19d96ab1f50c703982e843d44c35b18d19 --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "incorrect": 6355, + "exact match": 1188, + "semantically correct": 2222, + "invalid": 226, + "timeout": 6, + "runtime error": 3, + "correct": 3410 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/summary.txt b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..b40bc2e4bfbbdd2ee60baa107daecb39def1b847 --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/nondyn-rope-perturbed eval-ted --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 3410/10000, 34.100000% +Exact match: 1188/10000, 11.880000% +Semantically correct: 2222/10000, 22.220000% +Incorrect: 6355/10000, 63.550000% +Invalid: 226/10000, 2.260000% +Timeout: 6/10000, 0.060000% +Runtime error: 3/10000, 0.030000% diff --git a/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/trace_times.png b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..2c41e04d3a3455e2b29090b8c9d01c1e171f4fe6 Binary files /dev/null and b/models/adacos/nondyn-rope-perturbed/results/ltl-35-val10k-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope-perturbed/runs/Sep25_20-37-00_as02r1b12/events.out.tfevents.1727289422.as02r1b12.2809115.0 b/models/adacos/nondyn-rope-perturbed/runs/Sep25_20-37-00_as02r1b12/events.out.tfevents.1727289422.as02r1b12.2809115.0 new file mode 100644 index 0000000000000000000000000000000000000000..90d15e580c6658d1de6d19d846651fa80a32a78d --- /dev/null +++ b/models/adacos/nondyn-rope-perturbed/runs/Sep25_20-37-00_as02r1b12/events.out.tfevents.1727289422.as02r1b12.2809115.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4407ce6e2b21740eda522d96fed8fbaab2f1ab121e5401d8c614080b87713707 +size 46566 diff --git a/models/adacos/nondyn-rope/command-log.txt b/models/adacos/nondyn-rope/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..8bf798a67a7ff762808a4d8691e3d150520501e7 --- /dev/null +++ b/models/adacos/nondyn-rope/command-log.txt @@ -0,0 +1,64 @@ +Number of parameters: 5_810_432 +Arguments: +{ + "model_path": "models/nondyn-rope", + "device": "cuda", + "seed": null, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": false, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/adacos/nondyn-rope/config.json b/models/adacos/nondyn-rope/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f14ff6cb6cc9c671f8628e0a6e7de08bf9b9fdae --- /dev/null +++ b/models/adacos/nondyn-rope/config.json @@ -0,0 +1,57 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": false, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd" + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope" +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope/eval2da.pkl b/models/adacos/nondyn-rope/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..4e6812acf331891133e6bd4de420d3ce47b5d724 --- /dev/null +++ b/models/adacos/nondyn-rope/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13f620700de147866b406f29f82a31ebbf995e2e2e096472197d8712d958fd90 +size 1589445 diff --git a/models/adacos/nondyn-rope/eval2da.png b/models/adacos/nondyn-rope/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..a14319d04796f761eb9f0a82632e51f5b2364a01 Binary files /dev/null and b/models/adacos/nondyn-rope/eval2da.png differ diff --git a/models/adacos/nondyn-rope/eval2da1.pkl b/models/adacos/nondyn-rope/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ca2cf28d9821db092ab859ff1d05168cdff5d063 --- /dev/null +++ b/models/adacos/nondyn-rope/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29a253c0dd3e2d7b0f90b8ebd4b96614986ab7cfad630fb27ee6f607f77ecd99 +size 1728052 diff --git a/models/adacos/nondyn-rope/eval2da1.png b/models/adacos/nondyn-rope/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..a3bab383a51275d625fd81ecbfae79b092be6667 Binary files /dev/null and b/models/adacos/nondyn-rope/eval2da1.png differ diff --git a/models/adacos/nondyn-rope/pytorch_model.bin b/models/adacos/nondyn-rope/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..55b8e9bb727fcf528c7a94cb87e7f3c7c7845805 --- /dev/null +++ b/models/adacos/nondyn-rope/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e12375bf4c0bebd019835dad5cb53a8e4d699602a2ad57bc4e19d8c8333fcdd6 +size 23377998 diff --git a/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/size_hist.png b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..120844736a05094c2f2fdb08b66f7dfa61004f1f Binary files /dev/null and b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/size_hist.svg b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..8d2c91455445599c355c873b7b3509fa85a24ae5 --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-25T22:50:55.306905 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/summary.json b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..91a591bdc7690418cdba4c0506d30eaaa72bffdc --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 8015, + "incorrect": 221, + "semantically correct": 1759, + "invalid": 2, + "timeout": 2, + "runtime error": 1, + "correct": 9774 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/summary.txt b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..22a5e7c3d1e281a73db5248eb292f6ee4bae7380 --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/nondyn-rope eval-ted --ds-name=ltl-35-perturbed --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 9774/10000, 97.740000% +Exact match: 8015/10000, 80.150000% +Semantically correct: 1759/10000, 17.590000% +Incorrect: 221/10000, 2.210000% +Invalid: 2/10000, 0.020000% +Timeout: 2/10000, 0.020000% +Runtime error: 1/10000, 0.010000% diff --git a/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/trace_times.png b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..60ca5e5d3f05c43ce43d124a75f15150b306bafc Binary files /dev/null and b/models/adacos/nondyn-rope/results/ltl-35-perturbed-val10k-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope/results/ltl-35-test-b3/size_hist.png b/models/adacos/nondyn-rope/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..fea9a0276d0c9e70bea833423abf099ff3d1cf65 Binary files /dev/null and b/models/adacos/nondyn-rope/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope/results/ltl-35-test-b3/size_hist.svg b/models/adacos/nondyn-rope/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..29111823e3279cb09d508e4f4560c22c4c22a5a1 --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-21T19:37:23.642534 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope/results/ltl-35-test-b3/summary.json b/models/adacos/nondyn-rope/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b177a4d434f832cd908fc0e5937d4f2aac18affe --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "semantically correct": 15001, + "exact match": 83223, + "incorrect": 1720, + "timeout": 23, + "invalid": 20, + "runtime error": 2, + "correct": 98224 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope/results/ltl-35-test-b3/summary.txt b/models/adacos/nondyn-rope/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..79b64c6e901add0d14513226e676b82f275037aa --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/nondyn-rope eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 98224/99989, 98.234806% +Exact match: 83223/99989, 83.232156% +Semantically correct: 15001/99989, 15.002650% +Incorrect: 1720/99989, 1.720189% +Timeout: 23/99989, 0.023003% +Invalid: 20/99989, 0.020002% +Runtime error: 2/99989, 0.002000% diff --git a/models/adacos/nondyn-rope/results/ltl-35-test-b3/trace_times.png b/models/adacos/nondyn-rope/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..dd7a4493d1fd309bab0b834f4cc78e1d671fbab6 Binary files /dev/null and b/models/adacos/nondyn-rope/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope/results/ltl-35-val-b3/size_hist.png b/models/adacos/nondyn-rope/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..f65e5900b16c867c4874785f81b8a81233fc5f4c Binary files /dev/null and b/models/adacos/nondyn-rope/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope/results/ltl-35-val-b3/size_hist.svg b/models/adacos/nondyn-rope/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..5f1af0ac189d2e5e26ae922f1d70185de1696253 --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-09-27T11:01:27.638556 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope/results/ltl-35-val-b3/summary.json b/models/adacos/nondyn-rope/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bdf94b8693395579559fd67886deba6f8bb4f9ad --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 83252, + "semantically correct": 14944, + "incorrect": 1756, + "timeout": 23, + "invalid": 12, + "runtime error": 2, + "correct": 98196 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope/results/ltl-35-val-b3/summary.txt b/models/adacos/nondyn-rope/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..fcd6dd2bd210c7d15fbf839ea775de7a06665f21 --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/nondyn-rope eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 98196/99989, 98.206803% +Exact match: 83252/99989, 83.261159% +Semantically correct: 14944/99989, 14.945644% +Incorrect: 1756/99989, 1.756193% +Timeout: 23/99989, 0.023003% +Invalid: 12/99989, 0.012001% +Runtime error: 2/99989, 0.002000% diff --git a/models/adacos/nondyn-rope/results/ltl-35-val-b3/trace_times.png b/models/adacos/nondyn-rope/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..0f490c733853f50dca3d8f709d98d7e117e98231 Binary files /dev/null and b/models/adacos/nondyn-rope/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/size_hist.png b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..543ef9920183ba7109600f751d8ea8dae40754a1 Binary files /dev/null and b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/size_hist.png differ diff --git a/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/size_hist.svg b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..236e353635c0aa446eb54d645771b7369e0bf107 --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/size_hist.svg @@ -0,0 +1,1723 @@ + + + + + + + + 2024-09-25T22:52:02.346754 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/summary.json b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..99698c8ff2dc939c16ea5ed9faa2d62df7e872b1 --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 8352, + "semantically correct": 1444, + "incorrect": 199, + "timeout": 2, + "invalid": 2, + "runtime error": 1, + "correct": 9796 +} \ No newline at end of file diff --git a/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/summary.txt b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..f365221350e8ce83304f4b6e23037ebafaacb0c2 --- /dev/null +++ b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/nondyn-rope eval-ted --max-samples=10000 --beam-size=3 + +EVALUATION SUMMARY +Correct: 9796/10000, 97.960000% +Exact match: 8352/10000, 83.520000% +Semantically correct: 1444/10000, 14.440000% +Incorrect: 199/10000, 1.990000% +Timeout: 2/10000, 0.020000% +Invalid: 2/10000, 0.020000% +Runtime error: 1/10000, 0.010000% diff --git a/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/trace_times.png b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..60ca5e5d3f05c43ce43d124a75f15150b306bafc Binary files /dev/null and b/models/adacos/nondyn-rope/results/ltl-35-val10k-b3/trace_times.png differ diff --git a/models/adacos/nondyn-rope/runs/Sep25_20-36-58_as02r2b27/events.out.tfevents.1727289420.as02r2b27.1165137.0 b/models/adacos/nondyn-rope/runs/Sep25_20-36-58_as02r2b27/events.out.tfevents.1727289420.as02r2b27.1165137.0 new file mode 100644 index 0000000000000000000000000000000000000000..10371f9592f2c62cfe52a1a16d5e2c0229e8164a --- /dev/null +++ b/models/adacos/nondyn-rope/runs/Sep25_20-36-58_as02r2b27/events.out.tfevents.1727289420.as02r2b27.1165137.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82b67a1f2c72723fe7dc1d330b7be78415759e992ac0b036f6f114258fe76463 +size 46536 diff --git a/models/adacos/s005-rope-limited-s42/command-log.txt b/models/adacos/s005-rope-limited-s42/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..b44b6122508d6744eb8c29ae0d3496269cb20fc7 --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/command-log.txt @@ -0,0 +1,66 @@ +Number of parameters: 5_810_432 +Arguments: +{ + "model_path": "models/adacos/s005-rope-limited-s42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": 80000, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 5, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA GeForce RTX 3090 +Python version: 3.10.14 (main, May 6 2024, 19:42:50) [GCC 11.2.0] +numpy version: 1.26.4 +torch version: 2.3.0 +transformers version: 4.40.2 + diff --git a/models/adacos/s005-rope-limited-s42/config.json b/models/adacos/s005-rope-limited-s42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..dae31165d7cfaad80062eb7e27b4086f18bd1cd5 --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/config.json @@ -0,0 +1,59 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 5 + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/adacos/s005-rope-limited-s42/pytorch_model.bin b/models/adacos/s005-rope-limited-s42/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..60f74b342f2d85c096b28ffa5b062ae8e9084c1b --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75053b4c59134e414bf9207ad314e3b7ac94eb89dfb733c73b4ab9b8a437f849 +size 23378322 diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/evaluation.json b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/evaluation.json new file mode 100644 index 0000000000000000000000000000000000000000..f6ede9e63a01f70d14aaf31f7f3994f43d136544 --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/evaluation.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd265c05d0b6700c654a7de1b5f5b80339528cc7b266acf1916bd4295f2120da +size 20108272 diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/size_hist.png b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..9500f6a17df4e45a7c895cdd8706101161fd04e4 Binary files /dev/null and b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/size_hist.svg b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..1c1f4aaa68643c30d80be8adbd8c3ef11469f083 --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-26T13:30:54.852638 + image/svg+xml + + + Matplotlib v3.8.4, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/summary.json b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..17d16533aba9052b4bc887877a06a23d3783e14e --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "semantically correct": 25347, + "exact match": 64139, + "incorrect": 10283, + "invalid": 148, + "timeout": 68, + "runtime error": 4, + "correct": 89486 +} \ No newline at end of file diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/summary.txt b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..477b7bdf300e079532446102964a6a18d829dea6 --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/s005-rope-limited-s42 eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 89486/99989, 89.495845% +Exact match: 64139/99989, 64.146056% +Semantically correct: 25347/99989, 25.349788% +Incorrect: 10283/99989, 10.284131% +Invalid: 148/99989, 0.148016% +Timeout: 68/99989, 0.068007% +Runtime error: 4/99989, 0.004000% diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/trace_times.png b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..6ef78e7e50cfe5eae68f2197795097f4579c87fa Binary files /dev/null and b/models/adacos/s005-rope-limited-s42/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/evaluation.json b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/evaluation.json new file mode 100644 index 0000000000000000000000000000000000000000..116ef0ec6c892a4ef8758ead26139c4e9d56ce01 --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/evaluation.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e106a5a7eab797e4809c4a0c45b5111585ca2f1a0293901e49b85c421a24cfd +size 20117373 diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/size_hist.png b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..9740005b00f3d6bf4c18e1eb0cd14bb3bbd1d843 Binary files /dev/null and b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/size_hist.svg b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..194642996a18720af7252e0c29c09a9e48e5cb6e --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-26T13:16:08.531611 + image/svg+xml + + + Matplotlib v3.8.4, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/summary.json b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fc4cb610599a823165a382ba7ff671ef18a8041c --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 64169, + "semantically correct": 25091, + "incorrect": 10522, + "invalid": 145, + "timeout": 60, + "runtime error": 2, + "correct": 89260 +} \ No newline at end of file diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/summary.txt b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..a8a6f3f8af4b2123c466254d73bbd30da46668f0 --- /dev/null +++ b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/s005-rope-limited-s42 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 89260/99989, 89.269820% +Exact match: 64169/99989, 64.176059% +Semantically correct: 25091/99989, 25.093760% +Incorrect: 10522/99989, 10.523158% +Invalid: 145/99989, 0.145016% +Timeout: 60/99989, 0.060007% +Runtime error: 2/99989, 0.002000% diff --git a/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/trace_times.png b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..8db655d311afda51078934264845bc9be18266ac Binary files /dev/null and b/models/adacos/s005-rope-limited-s42/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/adacos/s005-rope-perturbed-s42/command-log.txt b/models/adacos/s005-rope-perturbed-s42/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..be8ff4eda6d1d0c69208f26154abdf82e6cbf861 --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/command-log.txt @@ -0,0 +1,66 @@ +Number of parameters: 5_810_432 +Arguments: +{ + "model_path": "models/adacos/s005-rope-perturbed-s42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 5, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA GeForce RTX 3090 +Python version: 3.10.14 (main, May 6 2024, 19:42:50) [GCC 11.2.0] +numpy version: 1.26.4 +torch version: 2.3.0 +transformers version: 4.40.2 + diff --git a/models/adacos/s005-rope-perturbed-s42/config.json b/models/adacos/s005-rope-perturbed-s42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..dae31165d7cfaad80062eb7e27b4086f18bd1cd5 --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/config.json @@ -0,0 +1,59 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 5 + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/adacos/s005-rope-perturbed-s42/pytorch_model.bin b/models/adacos/s005-rope-perturbed-s42/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..6e6ebe4585e902e245354d21f7b7ffd6e95be584 --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00f02195461ae224bc438d61f420df19e1c5ca1b013f106804349c3cabf80e93 +size 23378322 diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/evaluation.json b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/evaluation.json new file mode 100644 index 0000000000000000000000000000000000000000..d814dcc1755ed95d93b8076adf836863a35f1c5a --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/evaluation.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fe4ffc384dd67fa7bebd2b5a34ab183638643656053f1366e63bb480b26dc7b +size 20144014 diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/size_hist.png b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..aa9bb10f11e9282ad32e1e9910d433d7436ad72d Binary files /dev/null and b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/size_hist.svg b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..a1c63551561e4ccce1821e5aa377638e6a7a544e --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-26T18:12:25.560255 + image/svg+xml + + + Matplotlib v3.8.4, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/summary.json b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..71372925cd88ca0297fcc765d8d4612a41057cf3 --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 77652, + "semantically correct": 20306, + "incorrect": 1908, + "invalid": 73, + "timeout": 47, + "runtime error": 3, + "correct": 97958 +} \ No newline at end of file diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/summary.txt b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..85c9beb040b352ffb0fcf32912ac413603d849ed --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/s005-rope-perturbed-s42 eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 97958/99989, 97.968777% +Exact match: 77652/99989, 77.660543% +Semantically correct: 20306/99989, 20.308234% +Incorrect: 1908/99989, 1.908210% +Invalid: 73/99989, 0.073008% +Timeout: 47/99989, 0.047005% +Runtime error: 3/99989, 0.003000% diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/trace_times.png b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..e7d762c80dab27508e3ca50122724bd1ff4f0300 Binary files /dev/null and b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/evaluation.json b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/evaluation.json new file mode 100644 index 0000000000000000000000000000000000000000..ea6445265176af8b2132d7b6fde0bfb4193310c7 --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/evaluation.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1322f54f05f46a10a3aae622dfc51d9d1a5cd812f164454f8c992e180ade846b +size 20156577 diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/size_hist.png b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..01a6a7ae2d8bb5216493f709ffe1bb0aabf5bb33 Binary files /dev/null and b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/size_hist.svg b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..fbad20c06239a582593abe2732bc7f204692754c --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-26T17:54:41.710529 + image/svg+xml + + + Matplotlib v3.8.4, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/summary.json b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bf740da84cd1cd0752b2d5a25a6585d0d757aeb3 --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 77469, + "incorrect": 1899, + "semantically correct": 20494, + "invalid": 86, + "timeout": 39, + "runtime error": 2, + "correct": 97963 +} \ No newline at end of file diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/summary.txt b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..4643486c96b911170a9cb87c5a0061d1062c388f --- /dev/null +++ b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/adacos/s005-rope-perturbed-s42 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 97963/99989, 97.973777% +Exact match: 77469/99989, 77.477523% +Semantically correct: 20494/99989, 20.496255% +Incorrect: 1899/99989, 1.899209% +Invalid: 86/99989, 0.086009% +Timeout: 39/99989, 0.039004% +Runtime error: 2/99989, 0.002000% diff --git a/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/trace_times.png b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..20376dd3727bbc91c9c2586bc6e3e3e6f55199ed Binary files /dev/null and b/models/adacos/s005-rope-perturbed-s42/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/command-log.txt b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d23b3c7ea29b1f57b042793a4f3a3775ac19b603 --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 5_809_850 +Arguments: +{ + "model_path": "models/ltl-5/d005-rop-bn0-fn1-ada1-s42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 5, + "ap_embed": "diagbor", + "embed_base_normalization": "disabled", + "embed_ap_normalization": "disabled", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/config.json b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c521ef6b95fca7662f8e375ae9a361921978d60d --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/config.json @@ -0,0 +1,59 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 5, + "ap_embed": "diagbor", + "base_normalization": "disabled", + "ap_normalization": "disabled", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da.pkl b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6a5f75befa6d248bb00b1719a1dff090b427e5b4 --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21ccd9df90cbff4a743e197194887677a3e61d59d4217f602465c72a5a402b73 +size 3075450 diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da.png b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..9880f0e1435fed180a9c212e08ea8617601b7838 Binary files /dev/null and b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da.png differ diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da1.pkl b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2eccc857d89e196dc741be7d65e1785e1002f010 --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dc58b32c12684e69de71932aa013dcbf0d683a7222f5c7089d45db2f7e52a38 +size 3750350 diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da1.png b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..e307176b7627582139dc7682cd1d5271b44ac3c7 Binary files /dev/null and b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/eval2da1.png differ diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/pytorch_model.bin b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..0fb42c9ad57854826f319fe0147f765434d4670b --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efe97987ccf2ae4d1409f0fe02cc034be382322bfb849e3c8546df330282b13d +size 23375630 diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.png b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..9e666d1b3eeea237a1e61d773ebb7987f62028e3 Binary files /dev/null and b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.svg b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..96b20614a8b45cb6cb94d5510bf21051bedccfc3 --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2025-03-30T18:16:13.114347 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b4e1fe9369b1480f8d7c21e0766b7a410dd69125 --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "incorrect": 75926, + "exact match": 3012, + "semantically correct": 15761, + "invalid": 5047, + "timeout": 248, + "runtime error": 6, + "correct": 18773 +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..51896b32b155ff50c26580fd028b610ecc4deb05 --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/d005-rop-bn0-fn1-ada1-s42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 18773/100000, 18.773000% +Exact match: 3012/100000, 3.012000% +Semantically correct: 15761/100000, 15.761000% +Incorrect: 75926/100000, 75.926000% +Invalid: 5047/100000, 5.047000% +Timeout: 248/100000, 0.248000% +Runtime error: 6/100000, 0.006000% diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..34302e3bcdf564216d3eef7ea5047e76f89b417c Binary files /dev/null and b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..ad5ac46cd6b6912a40e5a92eccd5b12ebf277e02 Binary files /dev/null and b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..eba68d0ca182489db09c64df5372c86ec4e4cc1b --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2025-03-30T18:25:56.221876 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a3f25e6bb66fff6ac2eae60bfb2e71e55b707e6e --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "incorrect": 69631, + "exact match": 6287, + "semantically correct": 21930, + "invalid": 2059, + "timeout": 79, + "runtime error": 3, + "correct": 28217 +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..b1c5fe95b7555e0598eaa1210d659bf6177a6877 --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/d005-rop-bn0-fn1-ada1-s42 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 28217/99989, 28.220104% +Exact match: 6287/99989, 6.287692% +Semantically correct: 21930/99989, 21.932413% +Incorrect: 69631/99989, 69.638660% +Invalid: 2059/99989, 2.059227% +Timeout: 79/99989, 0.079009% +Runtime error: 3/99989, 0.003000% diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..858326e32929ea605083f84e40ad9c2a4c3f2568 Binary files /dev/null and b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/runs/Mar30_15-34-13_as01r4b16/events.out.tfevents.1743341654.as01r4b16.2694374.0 b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/runs/Mar30_15-34-13_as01r4b16/events.out.tfevents.1743341654.as01r4b16.2694374.0 new file mode 100644 index 0000000000000000000000000000000000000000..cf4f0e99c0ad78b4a4a686072bf8e57acfcef77b --- /dev/null +++ b/models/ltl-5/d005-rop-bn0-fn1-ada1-s42/runs/Mar30_15-34-13_as01r4b16/events.out.tfevents.1743341654.as01r4b16.2694374.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d12b61d0df1c19a210f76295c969254eaad345b8f6f863cecf98c56315e24b +size 46656 diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/command-log.txt b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d0762eb616fc48bfcfa37247f04a669778749ff6 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 5_809_850 +Arguments: +{ + "model_path": "models/ltl-5/d005-rop-bn1-fn0-ada0-s42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": null, + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 5, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "disabled", + "feature_normalization": "disabled", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/config.json b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ee6c4895289313fae3a7028a73ff892ca9310172 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/config.json @@ -0,0 +1,59 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 5, + "ap_embed": "diagbor", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "disabled", + "feature_normalization": "disabled", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da.pkl b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..af549a5010aa2578d1043707122373dd1a5f1198 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2d61d35583d016cf1d442806d1b21cc2af1330923cf25ba67ca85eb970e3f05 +size 3087012 diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da.png b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..ecd0008e39fd91e13e823ef4b65d3d07bf18413a Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da1.pkl b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d15075278d0303dccd4e2eb963b14ac6962e2fa6 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19a9c4110d8105982741d0662276ba684c789475a2b8a6466e28058edda2c540 +size 3767277 diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da1.png b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..0b6fa9c204de3302d98850540807199db8499fc2 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/eval2da1.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/pytorch_model.bin b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..9d6845e3ab840312fb2ef0e75693ee616c96716d --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8734f5d27d0b7f2028ea1385849c1d01536c291c224d363ee078d5c383b47183 +size 23375630 diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.png b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..fa3a6efd101585dc0768306160827b38ff5ae524 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.svg b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..800b32904120d4d352f4d42c919b20f989daf26a --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2025-03-30T18:11:51.595644 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/summary.json b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..64036c508721ce2e34c8b5195f0343616799448c --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 69905, + "semantically correct": 20525, + "incorrect": 9471, + "timeout": 70, + "invalid": 24, + "runtime error": 5, + "correct": 90430 +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/summary.txt b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..5cf0c15ac01bda7cdcc483a680fe6e8493f35384 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/d005-rop-bn1-fn0-ada0-s42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 90430/100000, 90.430000% +Exact match: 69905/100000, 69.905000% +Semantically correct: 20525/100000, 20.525000% +Incorrect: 9471/100000, 9.471000% +Timeout: 70/100000, 0.070000% +Invalid: 24/100000, 0.024000% +Runtime error: 5/100000, 0.005000% diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/trace_times.png b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..620111737847fa4f881b19a8230b4e10ccd77c2c Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/size_hist.png b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..620de362fb8cc9d08b55b32f608b2112d4c8ea76 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/size_hist.svg b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..ae9ae626593a06a9f546727ae4565e00319bdd28 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2025-03-30T18:20:45.201368 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/summary.json b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..953e9743a9eaf34df5239bf6bd91b12e4ff7ddbc --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 76430, + "incorrect": 3912, + "semantically correct": 19623, + "timeout": 18, + "runtime error": 2, + "invalid": 4, + "correct": 96053 +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/summary.txt b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..1def0f9f7069f5c9746734049aa34164f9342485 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/d005-rop-bn1-fn0-ada0-s42 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 96053/99989, 96.063567% +Exact match: 76430/99989, 76.438408% +Semantically correct: 19623/99989, 19.625159% +Incorrect: 3912/99989, 3.912430% +Timeout: 18/99989, 0.018002% +Runtime error: 2/99989, 0.002000% +Invalid: 4/99989, 0.004000% diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/trace_times.png b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..3a6e4c6ada6825723cc6ea3ef179b9813483281a Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/runs/Mar30_15-33-56_as01r5b07/events.out.tfevents.1743341637.as01r5b07.4000951.0 b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/runs/Mar30_15-33-56_as01r5b07/events.out.tfevents.1743341637.as01r5b07.4000951.0 new file mode 100644 index 0000000000000000000000000000000000000000..10004d26a95d72d737fee31f3d0583c02acb0603 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn0-ada0-s42/runs/Mar30_15-33-56_as01r5b07/events.out.tfevents.1743341637.as01r5b07.4000951.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:064ad9c8579f5c4cbc44126acd6b82cd84172f2a62c2b3c5c8e3d329c73c21c3 +size 40349 diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/command-log.txt b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..1fea42336ef49a7dc502f6077ee519e970bd5b56 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/command-log.txt @@ -0,0 +1,67 @@ +Number of parameters: 5_809_850 +Arguments: +{ + "model_path": "models/ltl-5/d005-rop-bn1-fn1-ada0-s42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "vocab_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": null, + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 5, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "disabled", + "embed_scaling": "sqrtd", + "shuffle_aps": null, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/config.json b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..85425a00521a53d77469898db832acb0663d4b2a --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/config.json @@ -0,0 +1,59 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 5, + "ap_embed": "diagbor", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "disabled", + "embed_scaling": "sqrtd", + "shuffle_aps": null + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da.pkl b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a0860e9b490f83ca4e0ef31eef0edd74f4a1a5a8 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6542456c010e95f8e18a8f21c3b6a9bcad811a30f0dc091868f9c8208cdcd35 +size 3098852 diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da.png b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..07acd13411d4ac5b227410620829410a0f1446e6 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da1.pkl b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ac91935ea5343f43ed22ed75c1311c6a713f5cd1 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd70239fcb34351f51f20473c9db50ea4ac8c3c8e6ab2ae237a973c3132e420d +size 3782082 diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da1.png b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..d497387ae5c823f49ddbfae017c005e653f55007 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/eval2da1.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/pytorch_model.bin b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..644c88fe810d81ca327730bf05d3e83f1fe9012c --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:454ef738d0d5fe9eb8b19526b60a0f1aad324da88bee5fde17389894f8cf32fb +size 23375630 diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.png b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..83a8e34821268f753d235713b11022c3e732afc9 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.svg b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..caad08f7654b81c400d5652e660fe89d705d0f1e --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2025-03-30T18:13:11.105515 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/summary.json b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9afc789366f423812b9ca3c9cb1645233b45bf8c --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 70013, + "semantically correct": 20501, + "incorrect": 9380, + "timeout": 71, + "invalid": 30, + "runtime error": 5, + "correct": 90514 +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/summary.txt b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..83579a61c51bfdb001a7f1126ada6b17135df669 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/d005-rop-bn1-fn1-ada0-s42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 90514/100000, 90.514000% +Exact match: 70013/100000, 70.013000% +Semantically correct: 20501/100000, 20.501000% +Incorrect: 9380/100000, 9.380000% +Timeout: 71/100000, 0.071000% +Invalid: 30/100000, 0.030000% +Runtime error: 5/100000, 0.005000% diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/trace_times.png b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..2c06a390a0dc2f4dfba9d6bcec8a50f4d507ae47 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/size_hist.png b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..080ab38c5506aa70133170114ab8e2f81eda4757 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/size_hist.svg b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..0dcd6a40697bb8adf9c5861fdf49f16f3a8f0368 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2025-03-30T18:22:08.972894 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/summary.json b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0c3ef257cd84f034ef5fdd8f13d6d6c0da7b66c6 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 76035, + "incorrect": 4291, + "semantically correct": 19632, + "timeout": 23, + "invalid": 6, + "runtime error": 2, + "correct": 95667 +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/summary.txt b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..e2764a889efe6460f5846d15381af62f0a1364c3 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/d005-rop-bn1-fn1-ada0-s42 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 95667/99989, 95.677525% +Exact match: 76035/99989, 76.043365% +Semantically correct: 19632/99989, 19.634160% +Incorrect: 4291/99989, 4.291472% +Timeout: 23/99989, 0.023003% +Invalid: 6/99989, 0.006001% +Runtime error: 2/99989, 0.002000% diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/trace_times.png b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..df027977444184e909d8fcd4a1f3aad3b4154455 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/runs/Mar30_15-34-13_as02r3b32/events.out.tfevents.1743341654.as02r3b32.3274942.0 b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/runs/Mar30_15-34-13_as02r3b32/events.out.tfevents.1743341654.as02r3b32.3274942.0 new file mode 100644 index 0000000000000000000000000000000000000000..83c163317133549324799d555d6dbd25093e21bb --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada0-s42/runs/Mar30_15-34-13_as02r3b32/events.out.tfevents.1743341654.as02r3b32.3274942.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43cbc5e67c0ee08d6fd632a96da2cf142fe0c0b2783eb03d972a4e53324d6cd5 +size 40343 diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/command-log.txt b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..3ceb801ca8772bf156a2686bfda2e87710fffb98 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/command-log.txt @@ -0,0 +1,130 @@ +Number of parameters: 5_809_850 +Arguments: +{ + "model_path": "models/dyns/diagbor5-42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-10ap", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 5, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + +Number of parameters: 5_809_850 +Arguments: +{ + "model_path": "models/dyns/diagbor5-42", + "device": "cuda", + "seed": 42, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 5, + "ap_embed": "diagbor", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/config.json b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac98374e1fbc8cb4bc24ea33dd576010624fc21c --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/config.json @@ -0,0 +1,63 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 5, + "ap_embed": "diagbor", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd" + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/d005-rop-bn1-fn1-ada1-s42.out b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/d005-rop-bn1-fn1-ada1-s42.out new file mode 100644 index 0000000000000000000000000000000000000000..c845227f8594fce3f10317462342b5c3d063c5b3 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/d005-rop-bn1-fn1-ada1-s42.out @@ -0,0 +1,5763 @@ ++ python3 -m autoregltl.main --model-path=models/dyns/diagbor5-42 --seed=42 train-ted --ds-name=ltl-35-perturbed --num-heads=8 --d-embed-enc=128 --d-ff=1024 --num-layers=8 --batch-size=768 --epochs=50 --val-max-samples=10000 --merge-tokens=all --merged-vocab --embed-scaling=sqrtd --dynamic-aps --d_ap=5 --ap_embed=diagbor --feature-normalization=l2 --loss-fct=adacos --tree-pos-enc --dec-pe=rope +Manual Seed: 42 +Using CUDA device: NVIDIA H100 +Training model: models/dyns/diagbor5-42 +[get_dataset_vocab] Vocab size: 39 (merged tokens: all) +Number of parameters: 5_809_850 +Filtered out 0 samples + train.txt: 0%| | 0/799909 [00:00 + + + + + + + 2024-10-01T20:46:09.866817 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..367ff34563f6308f07f4fc3ed1048a5191d4d634 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 78839, + "semantically correct": 15674, + "incorrect": 4740, + "invalid": 698, + "timeout": 47, + "runtime error": 2, + "correct": 94513 +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..bc161a5ea64243cb6ad6302108faa3b3b82f2dd4 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/d005-rop-bn1-fn1-ada1-s42 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 94513/100000, 94.513000% +Exact match: 78839/100000, 78.839000% +Semantically correct: 15674/100000, 15.674000% +Incorrect: 4740/100000, 4.740000% +Invalid: 698/100000, 0.698000% +Timeout: 47/100000, 0.047000% +Runtime error: 2/100000, 0.002000% diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..66375e6de4f095723e908a0be9478b607f16c8c7 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.png b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..32ccaf54993d725be6fc3a5b5e2c22fb646f2c6e Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.svg b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..e58e4253d6f37779d58ec9e1b7158e5223b883d1 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-25T22:19:44.872971 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/summary.json b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..dbcbcc8932a9dd627dfb7404bf175064121a2abb --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "semantically correct": 21464, + "exact match": 74830, + "incorrect": 3405, + "invalid": 254, + "timeout": 32, + "runtime error": 4, + "correct": 96294 +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/summary.txt b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..2f08dddb19c7651b16a5a03eef2b4578ea9b42fa --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/d005-rop-bn1-fn1-ada1-s42 eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 96294/99989, 96.304594% +Exact match: 74830/99989, 74.838232% +Semantically correct: 21464/99989, 21.466361% +Incorrect: 3405/99989, 3.405375% +Invalid: 254/99989, 0.254028% +Timeout: 32/99989, 0.032004% +Runtime error: 4/99989, 0.004000% diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/trace_times.png b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..51e290ed6bf487c9caed4ddc8600263e164a48ed Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..8fde9e11be0f8fa513fde639ff70297ad6669a01 Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..4ea1769d26155f54a61846bee749279ce623aece --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-09-29T04:32:28.817150 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.json b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..22edbd10cfb6158806bf453b6b6d494045b771b2 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 74697, + "incorrect": 3470, + "semantically correct": 21530, + "invalid": 259, + "timeout": 31, + "runtime error": 2, + "correct": 96227 +} \ No newline at end of file diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..30fb7eb36c1adceb1420a4f96203168731b0aa10 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/dyns/diagbor5-42 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 96227/99989, 96.237586% +Exact match: 74697/99989, 74.705218% +Semantically correct: 21530/99989, 21.532369% +Incorrect: 3470/99989, 3.470382% +Invalid: 259/99989, 0.259028% +Timeout: 31/99989, 0.031003% +Runtime error: 2/99989, 0.002000% diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..3b3c64e6ffc8a1cdf8fe86531fc8d6cb4bb5942f Binary files /dev/null and b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/runs/Sep29_01-34-27_as03r5b09/events.out.tfevents.1727566469.as03r5b09.1059804.0 b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/runs/Sep29_01-34-27_as03r5b09/events.out.tfevents.1727566469.as03r5b09.1059804.0 new file mode 100644 index 0000000000000000000000000000000000000000..756c0a0482d3f0a69ff3705a9d821c3383c6aeb1 --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/runs/Sep29_01-34-27_as03r5b09/events.out.tfevents.1727566469.as03r5b09.1059804.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:399aa0211628119ccba5f45549de46421b4a42dd2cf57a9197c7f577b1f6d8cb +size 46603 diff --git a/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/runs/Sep29_01-35-07_as07r2b06/events.out.tfevents.1727566508.as07r2b06.1581787.0 b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/runs/Sep29_01-35-07_as07r2b06/events.out.tfevents.1727566508.as07r2b06.1581787.0 new file mode 100644 index 0000000000000000000000000000000000000000..dbc4beb73bdf459ddd6ce44d87f7940cb271638a --- /dev/null +++ b/models/ltl-5/d005-rop-bn1-fn1-ada1-s42/runs/Sep29_01-35-07_as07r2b06/events.out.tfevents.1727566508.as07r2b06.1581787.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd90189cff722ae74eecf2e97257868236e6eb97f90bbaaddf96a35dff430a24 +size 46578 diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/command-log.txt b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/command-log.txt new file mode 100644 index 0000000000000000000000000000000000000000..f1dd3de8eace7d54176327842627be28f4e1a858 --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/command-log.txt @@ -0,0 +1,66 @@ +Number of parameters: 5_811_072 +Arguments: +{ + "model_path": "models/ltl-5/s010-rop-bn1-fn1-ada1-s46", + "device": "cuda", + "seed": 46, + "subparser": "train-ted", + "data_dir": "data", + "ds_name": "ltl-35-perturbed", + "max_trace_length": -1, + "min_aps": null, + "max_aps": null, + "exact_aps": null, + "val_split": "val", + "merged_vocab": true, + "merge_tokens": "all", + "dynamic_aps": true, + "learning_rate": 0.001, + "lr_scheduler_type": "cosine", + "warmup_steps": 1000, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "max_grad_norm": 1.0, + "epochs": 50, + "batch_size": 768, + "grad_acc_steps": 1, + "logging_steps": 500, + "eval_steps": 3000, + "train_max_samples": null, + "val_max_samples": 10000, + "trace_max_samples": 100, + "dry": false, + "eval": false, + "resume": false, + "loss_fct": "adacos", + "num_heads": 8, + "d_embed_enc": 128, + "d_embed_dec": null, + "d_ff": 1024, + "ff_activation": "relu", + "num_layers": 8, + "dropout": 0.1, + "layer_norm_eps": 1e-06, + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false, + "tree_pos_enc": true, + "d_ap": 0, + "ap_embed": "randn", + "embed_base_normalization": "l2", + "embed_ap_normalization": "l2", + "embed_final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 10, + "action": "train", + "model_type": "ted", + "decoder_only": false +} +Using CUDA device: NVIDIA H100 +Python version: 3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0] +numpy version: 1.26.2 +torch version: 2.1.1+cu121 +transformers version: 4.39.3 + diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/config.json b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2207e4fb43d9b3624eb5f20859231aa70d20fde2 --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/config.json @@ -0,0 +1,59 @@ +{ + "vocab": { + "aps": [ + "a", + "b", + "c", + "d", + "e" + ], + "consts": [ + "0", + "1" + ], + "trace_ops": [ + "&", + "|", + "!" + ], + "ltl_ops": [ + "U", + "X", + "!", + "&", + "|" + ], + "merge_tokens": "all", + "dynamic_aps": true, + "use_start_token": true, + "use_pad_token": true, + "use_eos_token": true + }, + "d_embed_enc": 128, + "d_embed_dec": 128, + "d_ff": 1024, + "ff_activation": "relu", + "dropout": 0.1, + "num_heads": 8, + "num_layers": 8, + "layer_norm_eps": 1e-06, + "merged_embedder": { + "tie_embeddings": true, + "pad_vocab_size_multiple": 8, + "d_ap": 0, + "ap_embed": "randn", + "base_normalization": "l2", + "ap_normalization": "l2", + "final_normalization": "l2", + "feature_normalization": "l2", + "embed_scaling": "sqrtd", + "shuffle_aps": 10 + }, + "max_encode_length": 1024, + "max_decode_length": 1024, + "tree_pos_enc": true, + "datatype": "float32", + "enc_pe": "sinusoid", + "dec_pe": "rope", + "no_pe_cross_keys": false +} \ No newline at end of file diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da.pkl b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da.pkl new file mode 100644 index 0000000000000000000000000000000000000000..845949d68f36636e98fd84a9cfc1032635a532da --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b108f1602094b58cb58414ebb12d042377e65646d0b0b11a8bf96c7f250e47a +size 3144569 diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da.png b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da.png new file mode 100644 index 0000000000000000000000000000000000000000..f18db17470fc2c639ee7b4cd98e99158a975d106 Binary files /dev/null and b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da.png differ diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da1.pkl b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e29f682c7b50acd225dd848e2bbae1bccc213d56 --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ab463a4bc8c12a457204a9501e8a89b099377f07d0439e447fe20d0d8ebf3ec +size 3843977 diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da1.png b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da1.png new file mode 100644 index 0000000000000000000000000000000000000000..36c4e1d871dfdf30c63e531bf1f87124f6964575 Binary files /dev/null and b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/eval2da1.png differ diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/pytorch_model.bin b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..305df2cc92c397cd015caf8c7753c16dd99f9fda --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32cfcc8b666954a162677c522834ea1943e857d134d9ab95df5d8bee62a33884 +size 23380882 diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/size_hist.png b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..14983cee96fe91128ed85f3e681fd96d5e6572d8 Binary files /dev/null and b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/size_hist.png differ diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/size_hist.svg b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..9956297c4676be627de281234985a05c316e0d46 --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-10-02T04:38:01.830120 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/summary.json b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fc81ee81ddd08bf630d99ae9861eb982f66c1e4c --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 71576, + "semantically correct": 22326, + "incorrect": 5506, + "invalid": 517, + "timeout": 70, + "runtime error": 5, + "correct": 93902 +} \ No newline at end of file diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/summary.txt b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..cdd3541089d1f9f58c82471104a84775c549f595 --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/s010-rop-bn1-fn1-ada1-s46 eval-ted --ds-name=ltl-35-10ap --beam-size=3 + +EVALUATION SUMMARY +Correct: 93902/100000, 93.902000% +Exact match: 71576/100000, 71.576000% +Semantically correct: 22326/100000, 22.326000% +Incorrect: 5506/100000, 5.506000% +Invalid: 517/100000, 0.517000% +Timeout: 70/100000, 0.070000% +Runtime error: 5/100000, 0.005000% diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/trace_times.png b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..17ccd2958e420db889702b01a1c589ae51e76392 Binary files /dev/null and b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-10ap-val-b3/trace_times.png differ diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/size_hist.png b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..e3b98c57cb79823bd9bc6e99055d64098ebb5434 Binary files /dev/null and b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/size_hist.png differ diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/size_hist.svg b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..f3413382897c7376605f7c918ec0bd1068d7ac6e --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-11-22T19:44:19.635738 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/summary.json b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d341149953ca49dcf8fae73fff04a12098d4b7ce --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/summary.json @@ -0,0 +1,9 @@ +{ + "semantically correct": 20323, + "exact match": 77384, + "incorrect": 2145, + "invalid": 109, + "timeout": 24, + "runtime error": 4, + "correct": 97707 +} \ No newline at end of file diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/summary.txt b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..1e4af7205f6ccf959779648613b17ffdb4e29905 --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/s010-rop-bn1-fn1-ada1-s46 eval-ted --ds-name=ltl-35 --split=test --beam-size=3 + +EVALUATION SUMMARY +Correct: 97707/99989, 97.717749% +Exact match: 77384/99989, 77.392513% +Semantically correct: 20323/99989, 20.325236% +Incorrect: 2145/99989, 2.145236% +Invalid: 109/99989, 0.109012% +Timeout: 24/99989, 0.024003% +Runtime error: 4/99989, 0.004000% diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/trace_times.png b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..6efc6942b78c0870a940854e58a6aeb77871a852 Binary files /dev/null and b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-test-b3/trace_times.png differ diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/size_hist.png b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/size_hist.png new file mode 100644 index 0000000000000000000000000000000000000000..b3030990dd2bcfd5f08bc3ae0b1764ca843aaf18 Binary files /dev/null and b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/size_hist.png differ diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/size_hist.svg b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/size_hist.svg new file mode 100644 index 0000000000000000000000000000000000000000..9212782fe910d98483ab1f787eb0e0020eaf0905 --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/size_hist.svg @@ -0,0 +1,1833 @@ + + + + + + + + 2024-10-02T04:47:03.270660 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/summary.json b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..686780b4261ac79e02206c2442656ece5a61a812 --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/summary.json @@ -0,0 +1,9 @@ +{ + "exact match": 76981, + "semantically correct": 20677, + "incorrect": 2168, + "invalid": 143, + "timeout": 18, + "runtime error": 2, + "correct": 97658 +} \ No newline at end of file diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/summary.txt b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..afa05291edf4138c24f6c48a1d1198509606b3dd --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/summary.txt @@ -0,0 +1,11 @@ +Command Line Arguments: +--model-path=models/ltl-5/s010-rop-bn1-fn1-ada1-s46 eval-ted --beam-size=3 + +EVALUATION SUMMARY +Correct: 97658/99989, 97.668744% +Exact match: 76981/99989, 76.989469% +Semantically correct: 20677/99989, 20.679275% +Incorrect: 2168/99989, 2.168239% +Invalid: 143/99989, 0.143016% +Timeout: 18/99989, 0.018002% +Runtime error: 2/99989, 0.002000% diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/trace_times.png b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/trace_times.png new file mode 100644 index 0000000000000000000000000000000000000000..2bcc23df74f5b41fe2c038ec5f2f17ddef5e67f8 Binary files /dev/null and b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/results/ltl-35-val-b3/trace_times.png differ diff --git a/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/runs/Oct02_02-00-09_as04r5b12/events.out.tfevents.1727827211.as04r5b12.338140.0 b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/runs/Oct02_02-00-09_as04r5b12/events.out.tfevents.1727827211.as04r5b12.338140.0 new file mode 100644 index 0000000000000000000000000000000000000000..16cd1adf59e1fa624f45fb43ae7eb0bb3cf5fd50 --- /dev/null +++ b/models/ltl-5/s010-rop-bn1-fn1-ada1-s46/runs/Oct02_02-00-09_as04r5b12/events.out.tfevents.1727827211.as04r5b12.338140.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a0c7c98a290d569b7b338d0b6366abe4fa0ca1f7fb8b61b311e403c6602d744 +size 46640