upload fp16 models

added_tokens.json

@@ -0,0 +1 @@
+{"[MASK]": 128000}

config.json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "pretrained-models/deberta-v2-xxlarge",
+  "add_enhanced_decoder": true,
+  "architectures": [
+    "DebertaV2ForMultipleChoicePreTrain"
+  ],
+  "attention_head_size": 64,
+  "attention_probs_dropout_prob": 0.1,
+  "conv_act": "gelu",
+  "conv_kernel_size": 3,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 6144,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "mlp_hidden_size": 3072,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 48,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1536,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.15.0",
+  "type_vocab_size": 0,
+  "use_stable_embedding": false,
+  "vocab_size": 128100
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74d9f7ed01565f457923e0c26f1a86c21ffd30e7bfa694eded38971658b75639
+size 3539235155

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5598d5e96f339a8d980c15f9afd405a2e5e1be7db41de3ed13b0f03fac1e8c17
+size 2447305

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "split_by_punct": false, "sp_model_kwargs": {}, "vocab_type": "spm", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "pretrained-models/deberta-v2-xxlarge", "tokenizer_class": "DebertaV2Tokenizer"}

training_config.yaml

@@ -0,0 +1,96 @@
+train_file: wiki_erica_path/v7/union/train_distant.path_v7.train.0.pkl
+dev_file: wiki_erica_path/v7/union/train_distant.path_v7.dev.pkl
+test_file: null
+model:
+  _target_: models.deberta.DebertaV2ForMultipleChoicePreTrain.from_pretrained
+  mlp_hidden_size: 3072
+  fs_checkpoint: false
+  fs_checkpoint_offload_to_cpu: false
+read_tensor:
+  _target_: dataset.wiki_entity_path_v8_2.convert_examples_into_features
+  max_neg_num: 3
+  aug_num: 1
+  max_seq_length: 256
+  shuffle_context: true
+  min_rep_num: 5
+  geo_p: 0.4
+  deduct_ratio: 1.0
+  context_ratio: 1.0
+  num_workers: 32
+extended_vocab: null
+collator:
+  _target_: dataset.wiki_entity_path_v8.WikiPathDatasetCollatorWithContext
+  max_seq_length: 256
+  tokenizer: pretrained-models/deberta-v2-xxlarge
+  mlm_probability: 0.15
+  max_option_num: 4
+  swap: true
+num_workers: 4
+prefetch_factor: 4
+model_name_or_path: pretrained-models/deberta-v2-xxlarge
+pretrain: null
+output_dir: experiments/deberta.v2.xxlarge.path.v7_v8.2.2.1aug.ctx.A100.v1.3.w4.s${seed}.fsdp.adamw
+do_train: Train
+evaluate_during_training: true
+do_eval: false
+eval_sub_path: null
+do_preprocess: false
+per_gpu_train_batch_size: 2
+per_gpu_eval_batch_size: 2
+learning_rate: 1.0e-05
+gradient_accumulation_steps: 512
+weight_decay: 0.01
+adam_epsilon: 1.0e-06
+adam_betas: (0.9, 0.999)
+max_grad_norm: 1.0
+num_train_epochs: 1
+max_steps: 200
+warmup_proportion: 0.2
+warmup_steps: 0
+optimizer: null
+use_nvlamb: null
+bit_training: null
+multi_tensor: null
+logging_steps: 1
+save_steps: 50
+eval_steps: 50
+no_cuda: false
+seed: 42
+local_rank: 0
+fp16: true
+fp16_opt_level: O2
+ds_cfg:
+  train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
+  gradient_accumulation_steps: ${gradient_accumulation_steps}
+  optimizer:
+    type: AdamW
+    params:
+      lr: ${learning_rate}
+      betas:
+      - 0.9
+      - 0.999
+      eps: ${adam_epsilon}
+      weight_decay: ${weight_decay}
+  scheduler:
+    type: WarmupDecayLR
+    params:
+      total_num_steps: null
+      warmup_max_lr: ${learning_rate}
+      warmup_num_steps: null
+      warmup_type: linear
+  gradient_clipping: ${max_grad_norm}
+  fp16:
+    enabled: ${fp16}
+    initial_scale_power: 12
+  zero_optimization:
+    stage: 3
+  steps_per_print: 1024
+reshard_after_forward: false
+flatten_parameters: true
+move_grads_to_cpu: false
+move_params_to_cpu: false
+n_gpu: 1
+device: cuda:0
+train_batch_size: 2
+eval_batch_size: 2
+note: null