iweems / app.py
vojkas's picture
Inline LFS bootstrap to ensure data files are available on HF
65b9f7b
Raw
History Blame Contribute Delete
4.31 kB
# app.py
import os
from pathlib import Path
import subprocess
from collections import OrderedDict
import streamlit as st
# ------------------------------------------------------------------
# Detect if we're running inside a Hugging Face Space (best-effort)
# ------------------------------------------------------------------
IS_HF = any(k in os.environ for k in ("SPACE_ID", "HF_SPACE_ID", "HF_HOME", "HF_REPO_ID"))
# Opt-in flags (set in HF: Settings β†’ Variables & secrets)
SHOW_MAINTENANCE = os.environ.get("IWEEMS_MAINTENANCE", "0") == "1"
USE_LFS_BOOTSTRAP = IS_HF or (os.environ.get("IWEEMS_USE_LFS_BOOTSTRAP", "0") == "1")
# ------------------------------------------------------------------
# Optional maintenance panel (HF-only, and only when you enable it)
# ------------------------------------------------------------------
if IS_HF and SHOW_MAINTENANCE:
with st.expander("πŸ”§ Maintenance / Clear caches"):
# Hugging Face hub cache
if st.button("Clear Hugging Face hub cache"):
try:
from huggingface_hub import scan_cache_dir
report = scan_cache_dir().delete() # removes ~/.cache/huggingface/*
freed = getattr(report, "reclaimed_size", 0) / 1e6
st.success(f"Cleared hub cache (~{freed:.1f} MB).")
except Exception as e:
st.error(f"Hub cache clear failed: {e}")
# Streamlit caches
if st.button("Clear Streamlit caches"):
st.cache_data.clear()
st.cache_resource.clear()
st.success("Streamlit caches cleared. Please rerun.")
# ------------------------------------------------------------------
# LFS bootstrap (ensures real files, not pointer stubs) β€” HF only
# ------------------------------------------------------------------
def _is_lfs_pointer(p: Path) -> bool:
try:
if not p.exists() or p.stat().st_size > 300:
return False
return p.read_bytes().startswith(b"version https://git-lfs.github.com")
except Exception:
return False
def ensure_lfs_materialized(paths, repo_id: str = "vojkas/iweems"):
# Fast path: try git-lfs pull; ignore failures if not available
try:
subprocess.run(["git", "lfs", "install", "--local"], check=False,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
subprocess.run(["git", "lfs", "pull"], check=False,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except Exception:
pass
# Fallback: if any file still looks like a pointer, fetch actual bytes via hub
try:
from huggingface_hub import hf_hub_download
for rel in paths:
p = Path(rel)
if not p.exists() or _is_lfs_pointer(p):
real = hf_hub_download(repo_id=repo_id, filename=str(p))
p.parent.mkdir(parents=True, exist_ok=True)
p.write_bytes(Path(real).read_bytes())
except Exception:
# If hub isn't available, leave as-is; open() will surface a clear error.
pass
# Only do this on HF (or if you explicitly opt-in locally)
if USE_LFS_BOOTSTRAP:
ensure_lfs_materialized([
"data/vectors_dict_comp.pkl",
"data/coordinates3d_tsne_dict.pkl",
"data/filtered_vocab_df.json",
"data/cos_sim_matrices_dict_f16.pkl", # used by crosscorpora_comparisons
])
# ------------------------------------------------------------------
# Your original app wiring (unchanged)
# ------------------------------------------------------------------
import overview
import explorer
import crosscorpora_comparisons
st.set_page_config(
page_title="iWEEMS",
page_icon="πŸ“Š",
layout="wide",
)
PAGES = OrderedDict({
"πŸ“Š About": overview,
"πŸ” Visualization App": explorer,
"πŸ” Crosscorpora Comparisons": crosscorpora_comparisons,
})
DEFAULT_PAGE = "πŸ” Visualization App"
if "page" not in st.session_state:
st.session_state.page = DEFAULT_PAGE
st.sidebar.title("πŸ“Œ Navigation")
for idx, (page_name, module) in enumerate(PAGES.items()):
if st.sidebar.button(page_name, key=f"nav_btn_{idx}"):
st.session_state.page = page_name
selected_page = PAGES.get(st.session_state.page, PAGES[DEFAULT_PAGE])
selected_page.app()