taigasan commited on
Commit
82bc3a9
·
verified ·
1 Parent(s): 89a85d1

deploy app, storage, readme

Browse files
Files changed (1) hide show
  1. app.py +7 -27
app.py CHANGED
@@ -7,7 +7,7 @@ import html
7
  from pathlib import Path
8
 
9
  import pandas as pd
10
- import requests
11
 
12
  from storage import VoteStorage
13
 
@@ -17,37 +17,17 @@ RATINGS_APP_TOKEN = os.getenv("RATINGS_APP_TOKEN")
17
  POOL_REPO_ID = "taigasan/e6-visual-ratings"
18
  VOTE_STORAGE = VoteStorage(mode="local" if DEBUG_MODE else "hf", local_dir=LOCAL_DATA_DIR, token=RATINGS_APP_TOKEN)
19
 
20
-
21
- def _download_pool_parquet(repo_id: str, token: str | None) -> str:
22
- root = Path(__file__).resolve().parent / LOCAL_DATA_DIR
23
- root.mkdir(parents=True, exist_ok=True)
24
- out_path = root / "pool.parquet"
25
- tmp_path = out_path.with_suffix(".parquet.tmp")
26
- url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/pool.parquet"
27
- headers = {"Authorization": f"Bearer {token}"} if token else {}
28
- last_err = None
29
- for _ in range(3):
30
- try:
31
- with requests.get(url, headers=headers, stream=True, timeout=(10, 60)) as r:
32
- r.raise_for_status()
33
- with open(tmp_path, "wb") as f:
34
- for chunk in r.iter_content(chunk_size=1 << 20):
35
- if chunk:
36
- f.write(chunk)
37
- os.replace(tmp_path, out_path)
38
- return str(out_path)
39
- except Exception as e:
40
- last_err = e
41
- if out_path.exists():
42
- return str(out_path)
43
- raise RuntimeError(f"Failed to download pool.parquet from {repo_id}: {last_err}")
44
-
45
  # -- Pool dataset -----------------------------------------------------------
46
  if DEBUG_MODE:
47
  _pool_path = str(Path(__file__).resolve().parent / LOCAL_DATA_DIR / "pool.parquet")
48
  assert Path(_pool_path).exists(), f"Missing local debug pool file: {_pool_path}"
49
  else:
50
- _pool_path = _download_pool_parquet(POOL_REPO_ID, RATINGS_APP_TOKEN)
 
 
 
 
 
51
  _pool_df = pd.read_parquet(_pool_path)
52
  _pool_group_dfs = {g: gdf for g, gdf in _pool_df.groupby("group")}
53
 
 
7
  from pathlib import Path
8
 
9
  import pandas as pd
10
+ from huggingface_hub import hf_hub_download
11
 
12
  from storage import VoteStorage
13
 
 
17
  POOL_REPO_ID = "taigasan/e6-visual-ratings"
18
  VOTE_STORAGE = VoteStorage(mode="local" if DEBUG_MODE else "hf", local_dir=LOCAL_DATA_DIR, token=RATINGS_APP_TOKEN)
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  # -- Pool dataset -----------------------------------------------------------
21
  if DEBUG_MODE:
22
  _pool_path = str(Path(__file__).resolve().parent / LOCAL_DATA_DIR / "pool.parquet")
23
  assert Path(_pool_path).exists(), f"Missing local debug pool file: {_pool_path}"
24
  else:
25
+ _pool_path = hf_hub_download(
26
+ repo_id=POOL_REPO_ID,
27
+ filename="pool.parquet",
28
+ repo_type="dataset",
29
+ token=RATINGS_APP_TOKEN
30
+ )
31
  _pool_df = pd.read_parquet(_pool_path)
32
  _pool_group_dfs = {g: gdf for g, gdf in _pool_df.groupby("group")}
33