Upload folder using huggingface_hub

checkpoints/ep5-ba660-rank0.pt.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4fd53119b6054e52c071ff7fbb88f7e96a0e0452090629628ed6f6acebf9295
+size 13200865280

checkpoints/latest-rank0.pt.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4fd53119b6054e52c071ff7fbb88f7e96a0e0452090629628ed6f6acebf9295
+size 13200865280

checkpoints/train_config.yaml

@@ -0,0 +1,150 @@
+algorithms:
+  gradient_clipping:
+    clipping_threshold: 1.0
+    clipping_type: norm
+callbacks:
+  lr_monitor: {}
+  memory_monitor: {}
+  runtime_estimator: {}
+  speed_monitor:
+    window_size: 10
+console_log_interval: 50ba
+cross_doc_attention: false
+dataloaders:
+- dataset:
+    batch_type: lm
+    local: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/
+    max_seq_len: 2048
+    shuffle: true
+    split: train
+  drop_last: false
+  name: train_loader_docs
+  num_workers: 0
+- dataset:
+    batch_type: fact
+    local: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/qa
+    masking:
+      cross_doc_attention: false
+    max_seq_len: 2048
+    shuffle: true
+    split: qa_attribution_train
+  drop_last: false
+  name: train_q_a_url
+  num_workers: 0
+deepspeed_config:
+  bf16:
+    enabled: true
+  train_batch_size: 80
+  zero_optimization:
+    allgather_bucket_size: 200000000.0
+    contiguous_gradients: true
+    offload_optimizer:
+      device: cpu
+      pin_memory: true
+    overlap_comm: true
+    reduce_bucket_size: true
+    reduce_scatter: true
+    stage: 3
+device_eval_batch_size: 40
+device_train_microbatch_size: 2
+eval_first: false
+eval_interval: 1
+eval_subset_num_batches: -1
+experiment:
+  data:
+    augment:
+      doc:
+        do: false
+        method: permute
+        n_sample_per_doc: 2
+    finetune:
+      neg_create_probability: 0.0
+      number_non_attributable_negatives: 0
+    qa_data_path: /root/autodl-tmp/intrinsic-source-citation/dataset/ours
+    text_data_path: dataset/ours/pretrain
+    train_data_path: /root/autodl-tmp/intrinsic-source-citation/dataset/ours/pretrain/train
+  eval:
+    disable_all_eval: true
+    disable_attribution_eval: true
+    disable_non_attrib_eval: true
+    disable_qa_eval: true
+    icl_eval: false
+    ppl_eval: false
+    use_ais: false
+  experiment:
+    name: arxiv-citation-doc-id-begin
+    output_dir: outputs/experiments/
+  model:
+    checkpoint: /root/autodl-tmp/intrinsic-source-citation/outputs/experiments/arxiv-citation-doc-id-begin/checkpoints/latest-rank0.pt.tar
+    ckpt_dir: /root/autodl-tmp/intrinsic-source-citation/outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+    name: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
+  train:
+    config_template_path: conf/templates/train_config.yaml
+    cross_doc_attention: false
+    device_eval_batch_size: 40
+    device_train_microbatch_size: 2
+    eval_first: false
+    finetune_q_a: false
+    finetune_q_a_doc_url: false
+    finetune_q_a_url: true
+    finetune_q_url_a: false
+    loss_type: mask
+    lr: 8.0e-05
+    max_duration: 5ep
+    pretrain: true
+    q_a_url_predict_url_only: false
+    repeat_url_across_doc: false
+    save_folder: outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+    sequential: false
+    url_location: no_url
+    url_loss_factor: 1.0
+    weight_decay: 0.02
+global_seed: 17
+global_train_batch_size: 80
+log_to_console: true
+loggers:
+  wandb:
+    project: intrinsic-source-citation
+max_duration: 5ep
+max_seq_len: 2048
+model:
+  ckpt_dir: outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+  loss:
+    type: mask
+    url_loss_factor: 1.0
+  name: hf_causal_lm
+  pretrained: true
+  pretrained_model_name_or_path: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
+ood_url_trie: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/unseen_url_trie.pkl
+optimizer:
+  betas:
+  - 0.9
+  - 0.98
+  eps: 1.0e-06
+  lr: 8.0e-05
+  name: deepspeed_adam
+  weight_decay: 0.02
+precision: amp_bf16
+progress_bar: false
+run_name: arxiv-citation-doc-id-begin
+save_folder: outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+save_interval: 1ep
+save_num_checkpoints_to_keep: 1
+scheduler:
+  alpha_f: 0.1
+  name: linear_decay_with_warmup
+  t_warmup: 1ep
+seed: 17
+streaming: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/
+text_data_path: dataset/ours/pretrain
+tokenizer:
+  kwargs:
+    model_max_length: 2048
+  name: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming//tokenizer
+tokenizer_name: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming//tokenizer
+url_trie: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/url_trie.pkl
+dist_timeout: 600.0
+n_gpus: 1
+device_train_batch_size: 80
+device_train_grad_accum: 40
+n_params: 1100056576

data/streaming/qa/qa_attribution_train/shard.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb386042ae42c9046ed625fb9325cf6e7257670bbd774c86c396a1451944e0ea
+size 84003105

data/streaming/tokenizer/added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "</url>": 32001,
+  "<url>": 32000
+}

data/streaming/tokenizer/special_tokens_map.json

@@ -0,0 +1,40 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<url>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</url>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

data/streaming/tokenizer/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

data/streaming/tokenizer/tokenizer_config.json

@@ -0,0 +1,59 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "<url>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "</url>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<url>",
+    "</url>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}

data/streaming/train/shard.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:611d90f3ee5a2a6f7143c3382831f11b5232feb8da989d4667d3a10f7425c4d2
+size 75663055

data/streaming/unseen_url_trie.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e397f686e6d6595bfdc7f101f4e0e1e0402b78331e058ec31332dcdefff82ae
+size 5018418

data/streaming/url_trie.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48ef08365be9d614aaae9c7fb1e5db648a8b3dc33060e9129a0d3738bc3dc61a
+size 5021235

experiment_config.yaml

@@ -0,0 +1,48 @@
+data:
+  augment:
+    doc:
+      do: false
+      method: permute
+      n_sample_per_doc: 2
+  finetune:
+    neg_create_probability: 0.0
+    number_non_attributable_negatives: 0
+  qa_data_path: /root/autodl-tmp/intrinsic-source-citation/dataset/ours
+  text_data_path: dataset/ours/pretrain
+  train_data_path: /root/autodl-tmp/intrinsic-source-citation/dataset/ours/pretrain/train
+eval:
+  disable_all_eval: true
+  disable_attribution_eval: true
+  disable_non_attrib_eval: true
+  disable_qa_eval: true
+  icl_eval: false
+  ppl_eval: false
+  use_ais: false
+experiment:
+  name: arxiv-citation-doc-id-begin
+  output_dir: outputs/experiments/
+model:
+  checkpoint: /root/autodl-tmp/intrinsic-source-citation/outputs/experiments/arxiv-citation-doc-id-begin/checkpoints/latest-rank0.pt.tar
+  ckpt_dir: /root/autodl-tmp/intrinsic-source-citation/outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+  name: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
+train:
+  config_template_path: conf/templates/train_config.yaml
+  cross_doc_attention: false
+  device_eval_batch_size: 40
+  device_train_microbatch_size: 2
+  eval_first: false
+  finetune_q_a: false
+  finetune_q_a_doc_url: false
+  finetune_q_a_url: true
+  finetune_q_url_a: false
+  loss_type: mask
+  lr: 8.0e-05
+  max_duration: 5ep
+  pretrain: true
+  q_a_url_predict_url_only: false
+  repeat_url_across_doc: false
+  save_folder: outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+  sequential: false
+  url_location: no_url
+  url_loss_factor: 1.0
+  weight_decay: 0.02

train_config.yaml

@@ -0,0 +1,147 @@
+algorithms:
+  gradient_clipping:
+    clipping_threshold: 1.0
+    clipping_type: norm
+callbacks:
+  lr_monitor: {}
+  memory_monitor: {}
+  runtime_estimator: {}
+  speed_monitor:
+    window_size: 10
+console_log_interval: 50ba
+cross_doc_attention: false
+dataloaders:
+- dataset:
+    batch_type: lm
+    local: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/
+    masking:
+      cross_doc_attention: false
+    max_seq_len: 2048
+    shuffle: true
+    split: train
+  drop_last: false
+  name: train_loader_docs
+  num_workers: 0
+- dataset:
+    batch_type: fact
+    local: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/qa
+    masking:
+      cross_doc_attention: false
+    max_seq_len: 2048
+    shuffle: true
+    split: qa_attribution_train
+  drop_last: false
+  name: train_q_a_url
+  num_workers: 0
+deepspeed_config:
+  bf16:
+    enabled: true
+  train_batch_size: 80
+  zero_optimization:
+    allgather_bucket_size: 200000000.0
+    contiguous_gradients: true
+    offload_optimizer:
+      device: cpu
+      pin_memory: true
+    overlap_comm: true
+    reduce_bucket_size: true
+    reduce_scatter: true
+    stage: 3
+device_eval_batch_size: 40
+device_train_microbatch_size: 2
+eval_first: false
+eval_interval: 1
+eval_subset_num_batches: -1
+experiment:
+  data:
+    augment:
+      doc:
+        do: false
+        method: permute
+        n_sample_per_doc: 2
+    finetune:
+      neg_create_probability: 0.0
+      number_non_attributable_negatives: 0
+    qa_data_path: /root/autodl-tmp/intrinsic-source-citation/dataset/ours
+    text_data_path: dataset/ours/pretrain
+    train_data_path: /root/autodl-tmp/intrinsic-source-citation/dataset/ours/pretrain/train
+  eval:
+    disable_all_eval: true
+    disable_attribution_eval: true
+    disable_non_attrib_eval: true
+    disable_qa_eval: true
+    icl_eval: false
+    ppl_eval: false
+    use_ais: false
+  experiment:
+    name: arxiv-citation-doc-id-begin
+    output_dir: outputs/experiments/
+  model:
+    checkpoint: /root/autodl-tmp/intrinsic-source-citation/outputs/experiments/arxiv-citation-doc-id-begin/checkpoints/latest-rank0.pt.tar
+    ckpt_dir: /root/autodl-tmp/intrinsic-source-citation/outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+    name: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
+  train:
+    config_template_path: conf/templates/train_config.yaml
+    cross_doc_attention: false
+    device_eval_batch_size: 40
+    device_train_microbatch_size: 2
+    eval_first: false
+    finetune_q_a: false
+    finetune_q_a_doc_url: false
+    finetune_q_a_url: true
+    finetune_q_url_a: false
+    loss_type: mask
+    lr: 8.0e-05
+    max_duration: 5ep
+    pretrain: true
+    q_a_url_predict_url_only: false
+    repeat_url_across_doc: false
+    save_folder: outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+    sequential: false
+    url_location: no_url
+    url_loss_factor: 1.0
+    weight_decay: 0.02
+global_seed: 17
+global_train_batch_size: 80
+log_to_console: true
+loggers:
+  wandb:
+    project: intrinsic-source-citation
+max_duration: 5ep
+max_seq_len: 2048
+model:
+  ckpt_dir: outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+  loss:
+    type: mask
+    url_loss_factor: 1.0
+  name: hf_causal_lm
+  pretrained: true
+  pretrained_model_name_or_path: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
+ood_url_trie: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/unseen_url_trie.pkl
+optimizer:
+  betas:
+  - 0.9
+  - 0.98
+  eps: 1.0e-06
+  lr: 8.0e-05
+  name: deepspeed_adam
+  weight_decay: 0.02
+precision: amp_bf16
+progress_bar: false
+run_name: arxiv-citation-doc-id-begin
+save_folder: outputs/experiments/arxiv-citation-doc-id-begin/checkpoints
+save_interval: 1ep
+save_num_checkpoints_to_keep: 1
+scheduler:
+  alpha_f: 0.1
+  name: linear_decay_with_warmup
+  t_warmup: 1ep
+seed: 17
+streaming: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/
+text_data_path: dataset/ours/pretrain
+tokenizer:
+  kwargs:
+    model_max_length: 2048
+  name: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming//tokenizer
+tokenizer_name: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming//tokenizer
+url_trie: outputs/experiments/arxiv-citation-doc-id-begin/data/streaming/url_trie.pkl