File size: 1,233 Bytes
d899b9f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
import requests
import os
from tqdm import tqdm
links = []
with open('links.txt', 'r', encoding='utf-8') as file:
for line in file:
links.append(line.strip())
download_dir = "../datasets/roboset"
os.makedirs(download_dir, exist_ok=True)
for link in links:
filename = os.path.basename(link)
filepath = os.path.join(download_dir, filename)
print(f"Downloading {filename} from {link}")
response = requests.get(link, stream=True)
total_size_in_bytes = int(response.headers.get('content-length', 0))
block_size = 1024
if os.path.exists(filepath):
local_size = os.path.getsize(filepath)
if local_size == total_size_in_bytes:
print(f"{filename} already exists")
continue
progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
with open(filepath, 'wb') as f:
for data in response.iter_content(block_size):
progress_bar.update(len(data))
f.write(data)
progress_bar.close()
if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
print("ERROR, something went wrong")
print(f"Downloaded {filename}")
print("All files processed.")
|