Spaces:
Runtime error
Runtime error
Update run.py
Browse files
run.py
CHANGED
|
@@ -38,7 +38,13 @@ def fast_caption(sys_prompt, usr_prompt, temp, top_p, max_tokens, model, key, en
|
|
| 38 |
progress_info.append('Begin processing Hugging Face dataset.')
|
| 39 |
os.environ["TOKIO_WORKER_THREADS"] = "8"
|
| 40 |
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
| 41 |
-
pqfile = hf_hub_download(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
| 43 |
pf = pq.ParquetFile(pqfile)
|
| 44 |
for batch in pf.iter_batches(1):
|
|
|
|
| 38 |
progress_info.append('Begin processing Hugging Face dataset.')
|
| 39 |
os.environ["TOKIO_WORKER_THREADS"] = "8"
|
| 40 |
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
| 41 |
+
pqfile = hf_hub_download(
|
| 42 |
+
repo_id=video_hf,
|
| 43 |
+
filename='data/' + str(parquet_index).zfill(6) + '.parquet',
|
| 44 |
+
repo_type="dataset",
|
| 45 |
+
local_dir="/dev/shm",
|
| 46 |
+
token=video_hf_auth,
|
| 47 |
+
)
|
| 48 |
|
| 49 |
pf = pq.ParquetFile(pqfile)
|
| 50 |
for batch in pf.iter_batches(1):
|