Hug0endob commited on
Commit
95cc9d8
·
verified ·
1 Parent(s): 8d7e1df

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +20 -5
app.py CHANGED
@@ -51,18 +51,33 @@ def safe_head(url: str, timeout: int = 6):
51
  except Exception:
52
  return None
53
 
54
- def fetch_bytes(src: str, timeout: int = 60) -> bytes:
 
55
  if is_remote(src):
56
  head = safe_head(src)
57
  if head is not None:
58
- r = requests.get(src, timeout=timeout)
59
- r.raise_for_status()
 
 
 
 
 
 
 
 
 
 
 
60
  return r.content
61
  else:
62
  if not os.path.exists(src):
63
- raise FileNotFoundError(f"Path does not exist: {src}")
 
64
  with open(src, "rb") as f:
65
- return f.read()
 
 
66
 
67
  def save_bytes_to_temp(b: bytes, suffix: str) -> str:
68
  fd, path = tempfile.mkstemp(suffix=suffix)
 
51
  except Exception:
52
  return None
53
 
54
+ def fetch_bytes(src: str, stream_threshold: int = STREAM_THRESHOLD, timeout: int = 60, progress=None) -> bytes:
55
+ if progress: progress(0.05, desc="Checking remote/local source...")
56
  if is_remote(src):
57
  head = safe_head(src)
58
  if head is not None:
59
+ cl = head.headers.get("content-length")
60
+ if cl and int(cl) > stream_threshold:
61
+ if progress: progress(0.1, desc="Streaming large remote file...")
62
+ with requests.get(src, timeout=timeout, stream=True) as r:
63
+ r.raise_for_status()
64
+ fd, p = tempfile.mkstemp()
65
+ os.close(fd)
66
+ with open(p, "wb") as fh:
67
+ for chunk in r.iter_content(8192):
68
+ if chunk: fh.write(chunk)
69
+ with open(p, "rb") as fh: return fh.read()
70
+ r = safe_get(src, timeout=timeout)
71
+ if progress: progress(0.25, desc="Downloaded remote content")
72
  return r.content
73
  else:
74
  if not os.path.exists(src):
75
+ raise FileNotFoundError(f"Local path does not exist: {src}")
76
+ if progress: progress(0.05, desc="Reading local file...")
77
  with open(src, "rb") as f:
78
+ data = f.read()
79
+ if progress: progress(0.15, desc="Read local file")
80
+ return data
81
 
82
  def save_bytes_to_temp(b: bytes, suffix: str) -> str:
83
  fd, path = tempfile.mkstemp(suffix=suffix)