r3gm commited on
Commit
e3c7dda
·
verified ·
1 Parent(s): 8d72861

Update utils.py

Browse files
Files changed (1) hide show
  1. utils.py +35 -20
utils.py CHANGED
@@ -275,11 +275,11 @@ def civ_redirect_down(url, dir_, civitai_api_key, romanize, alternative_name):
275
  elif os.path.exists(os.path.join(dir_, filename_base)):
276
  return os.path.join(dir_, filename_base), filename_base
277
 
278
- aria2_command = (
279
- f'aria2c --console-log-level=error --summary-interval=10 -c -x 16 '
280
- f'-k 1M -s 16 -d "{dir_}" -o "{filename_base}" "{redirect_url}"'
281
  )
282
- r_code = os.system(aria2_command) # noqa
283
 
284
  # if r_code != 0:
285
  # raise RuntimeError(f"Failed to download file: {filename_base}. Error code: {r_code}")
@@ -293,27 +293,32 @@ def civ_redirect_down(url, dir_, civitai_api_key, romanize, alternative_name):
293
 
294
  def civ_api_down(url, dir_, civitai_api_key, civ_filename):
295
  """
296
- This method is susceptible to being blocked because it generates a lot of temp redirect links with aria2c.
297
- If an API key limit is reached, generating a new API key and using it can fix the issue.
298
  """
299
  output_path = None
300
-
301
  url_dl = url + f"?token={civitai_api_key}"
 
302
  if not civ_filename:
303
- aria2_command = f'aria2c -c -x 1 -s 1 -d "{dir_}" "{url_dl}"'
304
- os.system(aria2_command)
 
 
 
 
305
  else:
306
  output_path = os.path.join(dir_, civ_filename)
 
307
  if not os.path.exists(output_path):
308
- aria2_command = (
309
- f'aria2c --console-log-level=error --summary-interval=10 -c -x 16 '
310
- f'-k 1M -s 16 -d "{dir_}" -o "{civ_filename}" "{url_dl}"'
311
  )
312
- os.system(aria2_command)
313
-
314
  return output_path
315
 
316
-
317
  def drive_down(url, dir_):
318
  import gdown
319
 
@@ -354,10 +359,16 @@ def hf_down(url, dir_, hf_token, romanize):
354
  url = url.replace("/blob/", "/resolve/")
355
 
356
  if hf_token:
357
- user_header = f'"Authorization: Bearer {hf_token}"'
358
- os.system(f"aria2c --console-log-level=error --summary-interval=10 --header={user_header} -c -x 16 -k 1M -s 16 {url} -d {dir_} -o {filename}")
 
 
 
359
  else:
360
- os.system(f"aria2c --optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 {url} -d {dir_} -o {filename}")
 
 
 
361
 
362
  return output_path
363
 
@@ -370,7 +381,8 @@ def download_things(directory, url, hf_token="", civitai_api_key="", romanize=Fa
370
  downloaded_file_path = drive_down(url, directory)
371
  elif "huggingface.co" in url:
372
  downloaded_file_path = hf_down(url, directory, hf_token, romanize)
373
- elif "civitai.com" in url:
 
374
  if not civitai_api_key:
375
  msg = "You need an API key to download Civitai models."
376
  print(f"\033[91m{msg}\033[0m")
@@ -393,7 +405,10 @@ def download_things(directory, url, hf_token="", civitai_api_key="", romanize=Fa
393
  gr.Warning(msg)
394
  downloaded_file_path = civ_api_down(url, directory, civitai_api_key, civ_filename)
395
  else:
396
- os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
 
 
 
397
 
398
  return downloaded_file_path
399
 
 
275
  elif os.path.exists(os.path.join(dir_, filename_base)):
276
  return os.path.join(dir_, filename_base), filename_base
277
 
278
+ wget_command = (
279
+ f'wget -c -nv '
280
+ f'-O "{os.path.join(dir_, filename_base)}" "{redirect_url}"'
281
  )
282
+ r_code = os.system(wget_command) # noqa
283
 
284
  # if r_code != 0:
285
  # raise RuntimeError(f"Failed to download file: {filename_base}. Error code: {r_code}")
 
293
 
294
  def civ_api_down(url, dir_, civitai_api_key, civ_filename):
295
  """
296
+ This method is susceptible to being blocked because it generates a lot of temp redirect links with wget.
297
+ If an API key limit is reached, generating a new API key and using it can fix the issue. no
298
  """
299
  output_path = None
300
+
301
  url_dl = url + f"?token={civitai_api_key}"
302
+
303
  if not civ_filename:
304
+ wget_command = (
305
+ f'wget -c -nv '
306
+ f'-P "{dir_}" "{url_dl}"'
307
+ )
308
+ os.system(wget_command)
309
+
310
  else:
311
  output_path = os.path.join(dir_, civ_filename)
312
+
313
  if not os.path.exists(output_path):
314
+ wget_command = (
315
+ f'wget -c -nv '
316
+ f'-O "{output_path}" "{url_dl}"'
317
  )
318
+ os.system(wget_command)
319
+
320
  return output_path
321
 
 
322
  def drive_down(url, dir_):
323
  import gdown
324
 
 
359
  url = url.replace("/blob/", "/resolve/")
360
 
361
  if hf_token:
362
+ os.system(
363
+ f'wget -c -nv '
364
+ f'--header="Authorization: Bearer {hf_token}" '
365
+ f'-O "{os.path.join(dir_, filename)}" "{url}"'
366
+ )
367
  else:
368
+ os.system(
369
+ f'wget -c -nv '
370
+ f'-O "{os.path.join(dir_, filename)}" "{url}"'
371
+ )
372
 
373
  return output_path
374
 
 
381
  downloaded_file_path = drive_down(url, directory)
382
  elif "huggingface.co" in url:
383
  downloaded_file_path = hf_down(url, directory, hf_token, romanize)
384
+ elif "civitai." in url:
385
+ url = url.replace("civitai.red", "civitai.com")
386
  if not civitai_api_key:
387
  msg = "You need an API key to download Civitai models."
388
  print(f"\033[91m{msg}\033[0m")
 
405
  gr.Warning(msg)
406
  downloaded_file_path = civ_api_down(url, directory, civitai_api_key, civ_filename)
407
  else:
408
+ os.system(
409
+ f'wget -c -nv '
410
+ f'-P "{directory}" "{url}"'
411
+ )
412
 
413
  return downloaded_file_path
414