Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

data-00000-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0261da25432170e3bdd26ab4acf4fdb750bcf9a2ee2f57798e7c7e67403ff1a9
+size 497688368

data-00001-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4adf108728b2abdc849af1b7f228a1edb2a56d64d49317daf877df7b42742c61
+size 496777704

data-00002-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcc7668f9fc5f3d8b4047c0bf18b41545ccaa9482a9994a643ff5f2dc668e24c
+size 498509072

data-00003-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1979c93604aa1a66e48d8644e1427dcd4e5bc1e27529226d035c7553d8e8376
+size 498227808

data-00004-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:369f76b43136116ad05a9cb780d83c0ef4842075f2398bcd9a9e1c7c4e8eb73e
+size 498898992

data-00005-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc27b71fe3f1ca6b6fb48255864f4fee4fec3ceb9e403ccd3e30ca4d418f6fdd
+size 498098400

data-00006-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:861c6a2138ecac04124d3df2c8a98d4675e6a9f2e59fc086b7a905236d095337
+size 499679432

data-00007-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72265081c358f29d9e9b5931fdccd91a35a84ac432f0be2facfaa1ebd65c5461
+size 496757192

data-00008-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ba22e2a6ca10df498e682c0720c7b6ebe8f4ff6c6ca9c225f283834936cce49
+size 499448048

data-00009-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c75dea46231bdc25cf132f3c152c64738fdb7348afd3c258c1c910a150c70c22
+size 499060392

data-00010-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:933effd1858c531fa63a4232a9dd20822fd4db11719a5463b40142fc1db9e15c
+size 497749384

data-00011-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1eda3c35e1e0ab58b59c5cc8de60755431c6f9d0766386f032a7bc2bc88cf0c0
+size 498901472

data-00012-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f2f0f5f9d95c1a3d15ba694dada5ee419b795a13bc0029d851bf838baf83c57
+size 498590744

data-00013-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8944483ee6a6d3814a5da0099cbe65560686bd6715f2580067e97a1d49c07df
+size 497591096

data-00014-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca7d86708fc7fa8616ab1e0e939615bd237b89713e802b5a17b0bdd426ea1f40
+size 499301624

data-00015-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:073d358bbb554186ce8944768bff03c31b5a1c4c18c54727f92b40d1fce36816
+size 497506088

data-00016-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e445d504a51bc554ae718cbff7758ece82f13560db04ba9c84c656e29dcb24c
+size 499126352

data-00017-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2d9f7d5225c70241f4c523a6c1cf4daad8a0e6085c390ec419b5511bf69df71
+size 499393824

data-00018-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca2e0b263c02023f7f2ac8333b81c0d9d1d2143cda246f3e0bb3fd81e7327659
+size 498020704

data-00019-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe9c56c04e77385d9225e6dc7dee6417edd63d6e0e9afe0aab7cafbf0963f176
+size 498712960

data-00020-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94f939e45647cccaafe29e19d296a6414040eceb74d2d5950c811bdd8cfa61f1
+size 498645640

data-00021-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cc3e73f79f2f2ebf298a28fe7d0041bd3225487f2fa3491c4f930f1b5da2ab0
+size 496303240

data-00022-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:999d7bb9f239b28f48556da1e3dad819307b74ea7b7ae893b14707d4c4721ef5
+size 498762776

data-00023-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2752fbe1f1215d8454d3757c2e030baa8e7fa80cc0e9f26138730b24fdc15f57
+size 498696344

data-00024-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edb7bcb58abcd365ea364c8886041e42819cee2004e7f5703cf23eb7705ee5a3
+size 496880352

data-00025-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc3da599787bb9211111e68b3f4abbac0a2842f59c41199168b885d2d5e9bc67
+size 499286552

data-00026-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b570ebbd3924b35396b7584c3b60c95046747992a486d9caf8321b845e7d024
+size 498665104

data-00027-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b77af55b8a2c396b0c0750e40b5ba004ea345bf598ea1df5f57884f32133d8d
+size 499042680

data-00028-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e7ffdda486946daad3fbb4a8ab7800b596b4e99e34e1f597c3932b85ecfd475
+size 498485200

data-00029-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cbbbe1f60238834a1aae7ad17ba9846384080d0f5ba2edd86706cb3a7315ee1
+size 498240392

data-00030-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f1d67d59cfcc3d05d1da75a65e18ba5702559cf11546f97af9d2fefdcb63206
+size 497743504

data-00031-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a781c3bd39871dd8e8d820b696ebf668dd5bcd0edf1dd1c7fe17679dbf89d014
+size 499313096

data-00032-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10762bf05883cb76242484947d284cb7002265603c95165f857c173310ae9983
+size 498104528

data-00033-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16ae3f28032bb757616f1d1c69bf5156e5c1878097841c40304fb0f60cf33857
+size 500039224

data-00034-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08817affd44cb9dbabce20180042a83da299e26243cdd445ae670ff18fd4cd6b
+size 499120800

data-00035-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d21874a23d05f6ca664ae4c5eab17c9ceba13312644465a221a1564029ea57b
+size 497721712

data-00036-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88e3c89cc4eff1bd2f42d804ac8da2719f3df3d49b761e13b911f5ba82c8e59b
+size 499935112

data-00037-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80389e3def390c2f1405ff941c66fa7dda69046c27d488d6b76c2f81cc66fefd
+size 499458784

data-00038-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaa17ddd1758bea2f7bac84ac0b44d98334418d0366d6a36740b66edd879d404
+size 497678336

data-00039-of-00040.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:740fe81bc4aeb60ea9a56a9b839a86aa316575aad39c6cf0a3774dd073f4b1af
+size 496965912

dataset_info.json

@@ -0,0 +1,101 @@
+{
+  "builder_name": "parquet",
+  "citation": "",
+  "config_name": "default",
+  "dataset_name": "parquet",
+  "dataset_size": 5008753497,
+  "description": "",
+  "download_checksums": {
+    "/work/jijun/long_context_dataset/prod/offline_retrieval_dataset/train/reranked//narrativeqa/narrativeqa_rerank.parquet": {
+      "num_bytes": 2530916504,
+      "checksum": null
+    }
+  },
+  "download_size": 2530916504,
+  "features": {
+    "id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "dataset_name": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "chunk_size": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "top_k_starts_new": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "concatenated_text": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "input_ids": {
+      "feature": {
+        "dtype": "int32",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "labels": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "retrieve_idx": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "length": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "answer_pos": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 7539670001,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 5008753497,
+      "num_examples": 38203,
+      "shard_lengths": [
+        4000,
+        4000,
+        4000,
+        4000,
+        4000,
+        4000,
+        4000,
+        4000,
+        4000,
+        2203
+      ],
+      "dataset_name": "parquet"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}

state.json

@@ -0,0 +1,130 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00040.arrow"
+    },
+    {
+      "filename": "data-00001-of-00040.arrow"
+    },
+    {
+      "filename": "data-00002-of-00040.arrow"
+    },
+    {
+      "filename": "data-00003-of-00040.arrow"
+    },
+    {
+      "filename": "data-00004-of-00040.arrow"
+    },
+    {
+      "filename": "data-00005-of-00040.arrow"
+    },
+    {
+      "filename": "data-00006-of-00040.arrow"
+    },
+    {
+      "filename": "data-00007-of-00040.arrow"
+    },
+    {
+      "filename": "data-00008-of-00040.arrow"
+    },
+    {
+      "filename": "data-00009-of-00040.arrow"
+    },
+    {
+      "filename": "data-00010-of-00040.arrow"
+    },
+    {
+      "filename": "data-00011-of-00040.arrow"
+    },
+    {
+      "filename": "data-00012-of-00040.arrow"
+    },
+    {
+      "filename": "data-00013-of-00040.arrow"
+    },
+    {
+      "filename": "data-00014-of-00040.arrow"
+    },
+    {
+      "filename": "data-00015-of-00040.arrow"
+    },
+    {
+      "filename": "data-00016-of-00040.arrow"
+    },
+    {
+      "filename": "data-00017-of-00040.arrow"
+    },
+    {
+      "filename": "data-00018-of-00040.arrow"
+    },
+    {
+      "filename": "data-00019-of-00040.arrow"
+    },
+    {
+      "filename": "data-00020-of-00040.arrow"
+    },
+    {
+      "filename": "data-00021-of-00040.arrow"
+    },
+    {
+      "filename": "data-00022-of-00040.arrow"
+    },
+    {
+      "filename": "data-00023-of-00040.arrow"
+    },
+    {
+      "filename": "data-00024-of-00040.arrow"
+    },
+    {
+      "filename": "data-00025-of-00040.arrow"
+    },
+    {
+      "filename": "data-00026-of-00040.arrow"
+    },
+    {
+      "filename": "data-00027-of-00040.arrow"
+    },
+    {
+      "filename": "data-00028-of-00040.arrow"
+    },
+    {
+      "filename": "data-00029-of-00040.arrow"
+    },
+    {
+      "filename": "data-00030-of-00040.arrow"
+    },
+    {
+      "filename": "data-00031-of-00040.arrow"
+    },
+    {
+      "filename": "data-00032-of-00040.arrow"
+    },
+    {
+      "filename": "data-00033-of-00040.arrow"
+    },
+    {
+      "filename": "data-00034-of-00040.arrow"
+    },
+    {
+      "filename": "data-00035-of-00040.arrow"
+    },
+    {
+      "filename": "data-00036-of-00040.arrow"
+    },
+    {
+      "filename": "data-00037-of-00040.arrow"
+    },
+    {
+      "filename": "data-00038-of-00040.arrow"
+    },
+    {
+      "filename": "data-00039-of-00040.arrow"
+    }
+  ],
+  "_fingerprint": "6b7655e0b76f210e",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}