Spaces:
Running
Running
Fix: download FAISS from dataset repo (repo_type=dataset)
Browse files
app.py
CHANGED
|
@@ -10,29 +10,15 @@ import traceback
|
|
| 10 |
import gradio as gr
|
| 11 |
from huggingface_hub import hf_hub_download
|
| 12 |
import os, shutil
|
| 13 |
-
|
| 14 |
|
| 15 |
def ensure_faiss_index_present():
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
os.makedirs(base_dir, exist_ok=True)
|
| 19 |
-
|
| 20 |
-
faiss_path = os.path.join(base_dir, "index.faiss")
|
| 21 |
-
pkl_path = os.path.join(base_dir, "index.pkl")
|
| 22 |
-
|
| 23 |
-
# Déjà présent -> on ne retélécharge pas
|
| 24 |
-
if os.path.exists(faiss_path) and os.path.exists(pkl_path):
|
| 25 |
-
return
|
| 26 |
-
|
| 27 |
-
# Repo HF qui contient l'index (DATASET, pas model, pas space)
|
| 28 |
-
repo_id = os.getenv("FAISS_REPO_ID", "FabIndy/code-education-faiss-index")
|
| 29 |
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
if not token or not token.startswith("hf_"):
|
| 33 |
-
token = None
|
| 34 |
|
| 35 |
-
# Téléchargement depuis un dataset repo
|
| 36 |
f_faiss = hf_hub_download(
|
| 37 |
repo_id=repo_id,
|
| 38 |
repo_type="dataset",
|
|
@@ -46,9 +32,9 @@ def ensure_faiss_index_present():
|
|
| 46 |
token=token,
|
| 47 |
)
|
| 48 |
|
| 49 |
-
|
| 50 |
-
shutil.copyfile(
|
| 51 |
-
|
| 52 |
|
| 53 |
|
| 54 |
|
|
|
|
| 10 |
import gradio as gr
|
| 11 |
from huggingface_hub import hf_hub_download
|
| 12 |
import os, shutil
|
| 13 |
+
from pathlib import Path
|
| 14 |
|
| 15 |
def ensure_faiss_index_present():
|
| 16 |
+
repo_id = os.environ.get("FAISS_REPO_ID", "FabIndy/code-education-faiss-index")
|
| 17 |
+
token = os.environ.get("HF_TOKEN") # optionnel si dataset public
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
+
local_dir = Path("db/faiss_code_edu_by_article")
|
| 20 |
+
local_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
|
| 21 |
|
|
|
|
| 22 |
f_faiss = hf_hub_download(
|
| 23 |
repo_id=repo_id,
|
| 24 |
repo_type="dataset",
|
|
|
|
| 32 |
token=token,
|
| 33 |
)
|
| 34 |
|
| 35 |
+
shutil.copyfile(f_faiss, local_dir / "index.faiss")
|
| 36 |
+
shutil.copyfile(f_pkl, local_dir / "index.pkl")
|
| 37 |
+
|
| 38 |
|
| 39 |
|
| 40 |
|