Upload 4 files

config.json

@@ -0,0 +1,157 @@
+{
+  "src_vocab_size": 250880,
+  "report_every": 50,
+  "save_data": "/media/vincent/Crucial X6/NMT_work/en-de/runs/48-0-32-4096-16384/",
+  "skip_empty_level": "silent",
+  "decoder_start_token": "<s>",
+  "seed": 1234,
+  "log_file": "/media/vincent/Crucial X6/NMT_work/en-de/runs/48-0-32-4096-16384-with-estim/48-0-32-4096-16384-with-estim.log",
+  "n_sample": 0,
+  "tgt_vocab_size": 250880,
+  "default_specials": [
+    "<unk>",
+    "<blank>",
+    "<s>",
+    "</s>"
+  ],
+  "model": {
+    "rotary_theta": 10000,
+    "hidden_size": 4096,
+    "transformer_ff": 16384,
+    "layers": 48,
+    "parallel_residual": false,
+    "mlp_activation_fn": "gelu",
+    "add_ffnbias": true,
+    "add_qkvbias": true,
+    "norm_eps": 1e-05,
+    "heads": 32,
+    "embeddings": {
+      "n_positions": 514,
+      "word_vec_size": 4096,
+      "src_word_vec_size": 4096,
+      "position_shift": 2,
+      "freeze_word_vecs_enc": true,
+      "position_encoding_type": "Learned",
+      "tgt_word_vec_size": 4096,
+      "position_encoding": true
+    },
+    "shared_layer_norm": false,
+    "num_experts_per_tok": 0,
+    "max_relative_positions": 0,
+    "heads_kv": 32,
+    "num_experts": 0,
+    "architecture": "transformer_encoder",
+    "sliding_window": 0,
+    "share_decoder_embeddings": true,
+    "left_pad": false,
+    "add_estimator": true,
+    "encoder": {
+      "encoder_type": "transformer",
+      "src_word_vec_size": 4096
+    },
+    "layer_norm": "standard",
+    "rotary_interleave": false,
+    "rotary_dim": 0
+  },
+  "src_vocab": "/mnt/InternalCrucial4/LLM_work/xlm-roberta-xl-eole/dict2.txt",
+  "vocab_size_multiple": 1,
+  "share_vocab": true,
+  "tgt_vocab": null,
+  "transforms": [
+    "sentencepiece"
+  ],
+  "transforms_configs": {
+    "onmt_tokenize": {},
+    "tokendrop": {},
+    "bpe": {},
+    "filtertoolong": {
+      "src_seq_length": 94,
+      "tgt_seq_length": 94
+    },
+    "inlinetags": {},
+    "clean": {},
+    "suffix": {},
+    "docify": {},
+    "switchout": {},
+    "uppercase": {},
+    "terminology": {},
+    "sentencepiece": {
+      "tgt_subword_model": "/mnt/InternalCrucial4/LLM_work/xlm-roberta-xxl-eole/sentencepiece.bpe.model",
+      "src_subword_model": "/mnt/InternalCrucial4/LLM_work/xlm-roberta-xxl-eole/sentencepiece.bpe.model"
+    },
+    "normalize": {},
+    "bart": {},
+    "insert_mask_before_placeholder": {},
+    "prefix": {},
+    "tokenmask": {}
+  },
+  "training": {
+    "world_size": 1,
+    "w_bit": 0,
+    "group_size": 0,
+    "batch_type": "sents",
+    "param_init_glorot": true,
+    "prefetch_factor": 400,
+    "learning_rate_decay": 1.0,
+    "decay_steps": 100000,
+    "param_init": 0.0,
+    "save_checkpoint_steps": 4000,
+    "accum_count": [
+      8
+    ],
+    "num_workers": 2,
+    "model_dtype": "fp16",
+    "start_decay_steps": 1000000,
+    "label_smoothing": 0.1,
+    "keep_checkpoint": 50,
+    "train_from": "/mnt/InternalCrucial4/LLM_work/xlm-roberta-xxl-eole/",
+    "valid_batch_size": 1,
+    "estim_loss_lambda_steps": [
+      0
+    ],
+    "quant_type": "bnb_NF4",
+    "batch_size_multiple": 1,
+    "attention_dropout": [
+      0.0
+    ],
+    "learning_rate": 1.5e-05,
+    "model_path": "/media/vincent/Crucial X6/NMT_work/en-de/runs/48-0-32-4096-16384-with-estim",
+    "batch_size": 8,
+    "dropout_steps": [
+      0
+    ],
+    "dropout": [
+      0.1
+    ],
+    "score_threshold": 0.0,
+    "gpu_ranks": [
+      0
+    ],
+    "optim": "fusedadam",
+    "normalization": "tokens",
+    "valid_steps": 1000,
+    "train_steps": 4000,
+    "adam_beta2": 0.998,
+    "decay_method": "none",
+    "estim_loss_lambda": [
+      1.0
+    ],
+    "average_decay": 0.0,
+    "accum_steps": [
+      0
+    ],
+    "quant_layers": [
+      "linear_values",
+      "linear_query",
+      "linear_keys",
+      "final_linear",
+      "gate_up_proj",
+      "down_proj"
+    ],
+    "max_grad_norm": 1.0,
+    "self_attn_backend": "pytorch",
+    "freeze_encoder": true,
+    "bucket_size": 262144
+  },
+  "data": {}
+}

model.00.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:570c25be3f39defca5b96f768d5d1bfe5a3587460d1880873722d99d7e0064ef
+size 21423469914

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051