Upload folder using huggingface_hub

data-00000-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9494f3ab54ec3ab4073f8186bd72a117034832971700ab75a48ebc398711867
+size 487501080

data-00001-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86e8e6dc4e25ea5015d0523dccc0b4377df24cc2e962cd698eca05683cf12551
+size 491954016

data-00002-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6f5463a8b3bd66da446d6953d9c0bc7042c30f5d2adb7c845c20583cb2ecf86
+size 479718848

data-00003-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dfa986bb31341f679a4b223f8ba09b2470318e7033c9c887a280dc266f1455b
+size 488730624

data-00004-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74720da1ba0c956d8c10101e326c331a94210127d690a39488fbbbe5d6e6226a
+size 490776048

data-00005-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f164adf618798d144cb5db51d0d9474a13bd0587ee14f07ba01958e264564e65
+size 495879000

data-00006-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12612ae4e0b3221cbc37119d8f92ae32a41839d45ca359876e6222f5d4e5d843
+size 487081136

data-00007-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3109bd1058ee41368881adbee7be84156456b0ea5fb5eba133756414faee522
+size 485015744

data-00008-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8512c339916d3fc25217e7f12e97b8fee7364932b3735a6d37a46937e0bef695
+size 499956656

data-00009-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53660cc0c72ab77b5760510081227eff5799516ec78bb2654cbf30a088688372
+size 480309376

data-00010-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7540c54c9984695a09ce3bb9351d1af05fe051cd089446415d7dfc8797fe643c
+size 482949432

data-00011-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e3e393a3e64e40f7c813316c31d0fadf0dd1fbc4fde962df4cbca6ba3a2d44d
+size 492876712

data-00012-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7727132a3b48e2723b0f6251eabf5eda77875215a7500be5688e74007d5ed239
+size 487963840

data-00013-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecfcd0304f9b5d3c8f3170da9f4d88412c6e3ac2fe357d90c7bca12044be6515
+size 496644072

data-00014-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8ff3e47d98214b244aeca9158b6131a0475f4f782d81c5335f5c376a01b40c5
+size 489826848

data-00015-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7b708c94f0d15735e2400249e3e774094582676fb4baa9a5fa700164d22f58c
+size 483721248

data-00016-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:553cabac0c61ef753f99013378ede2794cd80ece74936484f9d57af02d0836b5
+size 500170848

data-00017-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faf37d9ad1ebf507dfa613a478a78705222b80b1fa971796a3f97dff0eef7d61
+size 491021784

data-00018-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10530dadd7af93b23c311ca84b2112cbf2794423032323c7b035189c968b2299
+size 482726944

data-00019-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e24597f9724fe8131ae24de02e3087ae0581ee55bb798d20b0f989e352ed457
+size 483869648

data-00020-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee739c54957f158dc95e962c2216feb972638768c714e4cf33c0634da169b42b
+size 478443832

data-00021-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fca3f3cff7699a2a6d07e9b06d8031ac49294369a4c731db7b173d38398dce66
+size 487069888

data-00022-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fcd4611aed6b6f9842425dd38dc2f4b4bc1f915e661af6756db5eb263821ac0
+size 485343920

data-00023-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0ef08f2a9e5618ec6e2907d5733fb17ffc378078b898a5f55a54675f892c3a3
+size 490252128

data-00024-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76bfe20c2a4d8c53eb146cecae0d7ca65fdc23cdc0d4103fd7648b33e877d43b
+size 480027440

data-00025-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ff28cfd9eba3e64713a4b1647253d009325e1b446dd1a8d0d3a1f3945536927
+size 481756280

data-00026-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a27e850c0d07ad4b216c392f5963afb2052a1c37f200940b38485672677ef956
+size 488331808

data-00027-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50ae107960dbbe17f880d23e0985e8bfc186a8930b9151d66384db7f171c5af4
+size 480714496

data-00028-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d42c5b0ce4ef2d98fc2c13dc8c869d06042cca7ed702160231ceb7814b11c58
+size 492508480

data-00029-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18b594f561cfd6099c3e2754ddfed327e27bc381d3930a2f63cca602abab9356
+size 486451752

data-00030-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce90a7363d9b1b222c2abc24c6e90288b049696dfa9b3743e5c55431b441fb5c
+size 490103480

data-00031-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:474818cd24f727fdea41d27beffac3d6029f00632c01224d2aca79c26784c4d8
+size 490307728

data-00032-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55d55c1bc7058af741eca5c7ee94ff9a0f809319a4283c17e0c967599ca0c527
+size 487202952

data-00033-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:659abd246dfae6a8c470c440ee6987970ec9b654d0433e8edc5996452e3a9387
+size 479283872

data-00034-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84de7a91e50547003d3f3422e141a10753e6a06f7d25eb2bc37bbacfe764776c
+size 482694400

data-00035-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:029a99f01cde258173cd32a8244c457bdaab6d83622e126b84749e4f8d2f9c73
+size 482945152

data-00036-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34210465230c08c8bc00f1392fb85e508d68dd7e4cb8b772b342384069cf4713
+size 483156824

data-00037-of-00038.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffcda05d94d87dd2d93bf689e62ba9bd6544cb0449b1b39407ded3ba0f75c8c0
+size 484386208

dataset_info.json

@@ -0,0 +1,214 @@
+{
+  "builder_name": "openwebtext",
+  "citation": "@misc{Gokaslan2019OpenWeb,\n  title={OpenWebText Corpus},\n  author={Aaron Gokaslan*, Vanya Cohen*, Ellie Pavlick, Stefanie Tellex},\n  howpublished{\\url{http://Skylion007.github.io/OpenWebTextCorpus}},\n  year={2019}\n}\n",
+  "config_name": "plain_text",
+  "dataset_name": "openwebtext",
+  "dataset_size": 39769491688,
+  "description": "An open-source replication of the WebText dataset from OpenAI.\n",
+  "download_checksums": {
+    "subsets/urlsf_subset00.tar": {
+      "num_bytes": 633047040,
+      "checksum": null
+    },
+    "subsets/urlsf_subset01.tar": {
+      "num_bytes": 628838400,
+      "checksum": null
+    },
+    "subsets/urlsf_subset02.tar": {
+      "num_bytes": 629125120,
+      "checksum": null
+    },
+    "subsets/urlsf_subset03.tar": {
+      "num_bytes": 627578880,
+      "checksum": null
+    },
+    "subsets/urlsf_subset04.tar": {
+      "num_bytes": 627189760,
+      "checksum": null
+    },
+    "subsets/urlsf_subset05.tar": {
+      "num_bytes": 630200320,
+      "checksum": null
+    },
+    "subsets/urlsf_subset06.tar": {
+      "num_bytes": 625612800,
+      "checksum": null
+    },
+    "subsets/urlsf_subset07.tar": {
+      "num_bytes": 625356800,
+      "checksum": null
+    },
+    "subsets/urlsf_subset08.tar": {
+      "num_bytes": 624629760,
+      "checksum": null
+    },
+    "subsets/urlsf_subset09.tar": {
+      "num_bytes": 625807360,
+      "checksum": null
+    },
+    "subsets/urlsf_subset10.tar": {
+      "num_bytes": 625172480,
+      "checksum": null
+    },
+    "subsets/urlsf_subset11.tar": {
+      "num_bytes": 625264640,
+      "checksum": null
+    },
+    "subsets/urlsf_subset12.tar": {
+      "num_bytes": 624445440,
+      "checksum": null
+    },
+    "subsets/urlsf_subset13.tar": {
+      "num_bytes": 628961280,
+      "checksum": null
+    },
+    "subsets/urlsf_subset14.tar": {
+      "num_bytes": 626708480,
+      "checksum": null
+    },
+    "subsets/urlsf_subset15.tar": {
+      "num_bytes": 620666880,
+      "checksum": null
+    },
+    "subsets/urlsf_subset16.tar": {
+      "num_bytes": 618752000,
+      "checksum": null
+    },
+    "subsets/urlsf_subset17.tar": {
+      "num_bytes": 619141120,
+      "checksum": null
+    },
+    "subsets/urlsf_subset18.tar": {
+      "num_bytes": 617789440,
+      "checksum": null
+    },
+    "subsets/urlsf_subset19.tar": {
+      "num_bytes": 619192320,
+      "checksum": null
+    },
+    "subsets/urlsf_subset20.tar": {
+      "num_bytes": 376709120,
+      "checksum": null
+    }
+  },
+  "download_size": 12880189440,
+  "features": {
+    "text": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "token_ids": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "counterfactual_ids": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "https://skylion007.github.io/OpenWebTextCorpus/",
+  "license": "",
+  "size_in_bytes": 52649681128,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 39769491688,
+      "num_examples": 8013769,
+      "shard_lengths": [
+        101000,
+        100000,
+        101000,
+        101000,
+        102000,
+        102000,
+        101000,
+        102000,
+        101000,
+        101000,
+        101000,
+        101000,
+        101000,
+        102000,
+        101000,
+        101000,
+        101000,
+        101000,
+        102000,
+        102000,
+        100000,
+        101000,
+        100000,
+        101000,
+        102000,
+        101000,
+        102000,
+        101000,
+        102000,
+        102000,
+        102000,
+        101000,
+        101000,
+        101000,
+        101000,
+        102000,
+        101000,
+        102000,
+        101000,
+        101000,
+        100000,
+        101000,
+        101000,
+        101000,
+        101000,
+        101000,
+        101000,
+        101000,
+        101000,
+        101000,
+        101000,
+        100000,
+        101000,
+        102000,
+        101000,
+        101000,
+        101000,
+        101000,
+        101000,
+        102000,
+        102000,
+        101000,
+        102000,
+        101000,
+        102000,
+        102000,
+        101000,
+        101000,
+        102000,
+        102000,
+        102000,
+        101000,
+        102000,
+        102000,
+        102000,
+        101000,
+        101000,
+        102000,
+        101000,
+        13769
+      ],
+      "dataset_name": "openwebtext"
+    }
+  },
+  "version": {
+    "version_str": "1.0.0",
+    "major": 1,
+    "minor": 0,
+    "patch": 0
+  }
+}

state.json

@@ -0,0 +1,124 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00038.arrow"
+    },
+    {
+      "filename": "data-00001-of-00038.arrow"
+    },
+    {
+      "filename": "data-00002-of-00038.arrow"
+    },
+    {
+      "filename": "data-00003-of-00038.arrow"
+    },
+    {
+      "filename": "data-00004-of-00038.arrow"
+    },
+    {
+      "filename": "data-00005-of-00038.arrow"
+    },
+    {
+      "filename": "data-00006-of-00038.arrow"
+    },
+    {
+      "filename": "data-00007-of-00038.arrow"
+    },
+    {
+      "filename": "data-00008-of-00038.arrow"
+    },
+    {
+      "filename": "data-00009-of-00038.arrow"
+    },
+    {
+      "filename": "data-00010-of-00038.arrow"
+    },
+    {
+      "filename": "data-00011-of-00038.arrow"
+    },
+    {
+      "filename": "data-00012-of-00038.arrow"
+    },
+    {
+      "filename": "data-00013-of-00038.arrow"
+    },
+    {
+      "filename": "data-00014-of-00038.arrow"
+    },
+    {
+      "filename": "data-00015-of-00038.arrow"
+    },
+    {
+      "filename": "data-00016-of-00038.arrow"
+    },
+    {
+      "filename": "data-00017-of-00038.arrow"
+    },
+    {
+      "filename": "data-00018-of-00038.arrow"
+    },
+    {
+      "filename": "data-00019-of-00038.arrow"
+    },
+    {
+      "filename": "data-00020-of-00038.arrow"
+    },
+    {
+      "filename": "data-00021-of-00038.arrow"
+    },
+    {
+      "filename": "data-00022-of-00038.arrow"
+    },
+    {
+      "filename": "data-00023-of-00038.arrow"
+    },
+    {
+      "filename": "data-00024-of-00038.arrow"
+    },
+    {
+      "filename": "data-00025-of-00038.arrow"
+    },
+    {
+      "filename": "data-00026-of-00038.arrow"
+    },
+    {
+      "filename": "data-00027-of-00038.arrow"
+    },
+    {
+      "filename": "data-00028-of-00038.arrow"
+    },
+    {
+      "filename": "data-00029-of-00038.arrow"
+    },
+    {
+      "filename": "data-00030-of-00038.arrow"
+    },
+    {
+      "filename": "data-00031-of-00038.arrow"
+    },
+    {
+      "filename": "data-00032-of-00038.arrow"
+    },
+    {
+      "filename": "data-00033-of-00038.arrow"
+    },
+    {
+      "filename": "data-00034-of-00038.arrow"
+    },
+    {
+      "filename": "data-00035-of-00038.arrow"
+    },
+    {
+      "filename": "data-00036-of-00038.arrow"
+    },
+    {
+      "filename": "data-00037-of-00038.arrow"
+    }
+  ],
+  "_fingerprint": "74d0c30dce8b356d",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}