Update README.md
Browse files
README.md
CHANGED
|
@@ -1,3 +1,247 @@
|
|
| 1 |
---
|
| 2 |
license: mit
|
| 3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
license: mit
|
| 3 |
---
|
| 4 |
+
|
| 5 |
+
## setup uv env
|
| 6 |
+
|
| 7 |
+
```
|
| 8 |
+
#!/bin/bash
|
| 9 |
+
# Set up RAM+ in an isolated venv with a compatible transformers version.
|
| 10 |
+
# recognize-anything uses APIs removed in transformers>=4.46, so we pin to 4.45.2.
|
| 11 |
+
set -e
|
| 12 |
+
|
| 13 |
+
VENV=".venv-ram"
|
| 14 |
+
WEIGHTS_DIR="models/ram_plus"
|
| 15 |
+
WEIGHTS_FILE="$WEIGHTS_DIR/ram_plus_swin_large_14m.pth"
|
| 16 |
+
|
| 17 |
+
# ---------------------------------------------------------------------------
|
| 18 |
+
# 1. Create venv
|
| 19 |
+
# ---------------------------------------------------------------------------
|
| 20 |
+
if [ ! -d "$VENV" ]; then
|
| 21 |
+
echo "Creating $VENV …"
|
| 22 |
+
uv venv "$VENV" --python 3.11
|
| 23 |
+
else
|
| 24 |
+
echo "$VENV already exists"
|
| 25 |
+
fi
|
| 26 |
+
|
| 27 |
+
PIP="$VENV/bin/pip"
|
| 28 |
+
|
| 29 |
+
# ---------------------------------------------------------------------------
|
| 30 |
+
# 2. Install PyTorch (reuse system CUDA version if available)
|
| 31 |
+
# ---------------------------------------------------------------------------
|
| 32 |
+
if ! "$VENV/bin/python" -c "import torch" 2>/dev/null; then
|
| 33 |
+
echo "Installing PyTorch …"
|
| 34 |
+
CUDA_VER=$(python3 -c "import torch; print('cu' + torch.version.cuda.replace('.',''))" 2>/dev/null || echo "cpu")
|
| 35 |
+
if [ "$CUDA_VER" = "cpu" ]; then
|
| 36 |
+
uv pip install --python "$VENV/bin/python" torch torchvision
|
| 37 |
+
else
|
| 38 |
+
uv pip install --python "$VENV/bin/python" torch torchvision \
|
| 39 |
+
--index-url "https://download.pytorch.org/whl/$CUDA_VER"
|
| 40 |
+
fi
|
| 41 |
+
fi
|
| 42 |
+
|
| 43 |
+
# ---------------------------------------------------------------------------
|
| 44 |
+
# 3. Install recognize-anything + pinned transformers
|
| 45 |
+
# ---------------------------------------------------------------------------
|
| 46 |
+
if ! "$VENV/bin/python" -c "import ram" 2>/dev/null; then
|
| 47 |
+
echo "Installing recognize-anything with transformers==4.45.2 …"
|
| 48 |
+
uv pip install --python "$VENV/bin/python" \
|
| 49 |
+
"transformers==4.45.2" \
|
| 50 |
+
timm pillow scipy fairscale \
|
| 51 |
+
git+https://github.com/xinyu1205/recognize-anything.git
|
| 52 |
+
fi
|
| 53 |
+
|
| 54 |
+
# ---------------------------------------------------------------------------
|
| 55 |
+
# 4. Install Streamlit in the RAM venv
|
| 56 |
+
# ---------------------------------------------------------------------------
|
| 57 |
+
if ! "$VENV/bin/python" -c "import streamlit" 2>/dev/null; then
|
| 58 |
+
echo "Installing streamlit …"
|
| 59 |
+
uv pip install --python "$VENV/bin/python" streamlit
|
| 60 |
+
fi
|
| 61 |
+
|
| 62 |
+
# ---------------------------------------------------------------------------
|
| 63 |
+
# 5. Download weights
|
| 64 |
+
# ---------------------------------------------------------------------------
|
| 65 |
+
mkdir -p "$WEIGHTS_DIR"
|
| 66 |
+
if [ ! -f "$WEIGHTS_FILE" ]; then
|
| 67 |
+
echo "Downloading RAM+ weights (~3 GB) …"
|
| 68 |
+
"$VENV/bin/python" -c "
|
| 69 |
+
from huggingface_hub import hf_hub_download
|
| 70 |
+
path = hf_hub_download(
|
| 71 |
+
repo_id='ma7583/ramplus',
|
| 72 |
+
filename='ram_plus_swin_large_14m.pth',
|
| 73 |
+
local_dir='$WEIGHTS_DIR',
|
| 74 |
+
)
|
| 75 |
+
print('Downloaded to:', path)
|
| 76 |
+
"
|
| 77 |
+
else
|
| 78 |
+
echo "Weights already present at $WEIGHTS_FILE"
|
| 79 |
+
fi
|
| 80 |
+
|
| 81 |
+
echo ""
|
| 82 |
+
echo "Setup complete. Run the app with:"
|
| 83 |
+
echo " $VENV/bin/streamlit run tests/explore_ram.py"
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
```
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
## run streamlit for demo
|
| 90 |
+
|
| 91 |
+
```
|
| 92 |
+
"""RAM+ Video Object Explorer — Streamlit app.
|
| 93 |
+
|
| 94 |
+
Usage:
|
| 95 |
+
streamlit run tests/explore_ram.py
|
| 96 |
+
"""
|
| 97 |
+
|
| 98 |
+
import json
|
| 99 |
+
import subprocess
|
| 100 |
+
import tempfile
|
| 101 |
+
from pathlib import Path
|
| 102 |
+
from PIL import Image
|
| 103 |
+
import streamlit as st
|
| 104 |
+
import torch
|
| 105 |
+
|
| 106 |
+
VIDEO_DIR = Path(".")
|
| 107 |
+
WEIGHTS = Path("models/ram_plus/ram_plus_swin_large_14m.pth")
|
| 108 |
+
CACHE_DIR = Path("tests/mbs/ram_tags")
|
| 109 |
+
IMAGE_SIZE = 384
|
| 110 |
+
|
| 111 |
+
st.set_page_config(page_title="RAM+ Explorer", layout="wide")
|
| 112 |
+
|
| 113 |
+
# ---------------------------------------------------------------------------
|
| 114 |
+
# Load model
|
| 115 |
+
# ---------------------------------------------------------------------------
|
| 116 |
+
|
| 117 |
+
@st.cache_resource(show_spinner="Loading RAM+ model…")
|
| 118 |
+
def load_model(weights: str):
|
| 119 |
+
from ram.models import ram_plus
|
| 120 |
+
import torchvision.transforms as T
|
| 121 |
+
|
| 122 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 123 |
+
model = ram_plus(pretrained=weights, image_size=IMAGE_SIZE, vit="swin_l")
|
| 124 |
+
model.eval().to(device)
|
| 125 |
+
|
| 126 |
+
transform = T.Compose([
|
| 127 |
+
T.Resize((IMAGE_SIZE, IMAGE_SIZE)),
|
| 128 |
+
T.ToTensor(),
|
| 129 |
+
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
|
| 130 |
+
])
|
| 131 |
+
return model, transform, device
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
if not WEIGHTS.exists():
|
| 135 |
+
st.error(f"Weights not found at `{WEIGHTS}`. Run `bash run_ram_setup.sh` first.")
|
| 136 |
+
st.stop()
|
| 137 |
+
|
| 138 |
+
model, transform, device = load_model(str(WEIGHTS))
|
| 139 |
+
|
| 140 |
+
# ---------------------------------------------------------------------------
|
| 141 |
+
# Helpers
|
| 142 |
+
# ---------------------------------------------------------------------------
|
| 143 |
+
|
| 144 |
+
def extract_frames(video_path: Path, fps: float) -> list[tuple[float, Path]]:
|
| 145 |
+
tmp = Path(tempfile.mkdtemp(prefix="ram-frames-"))
|
| 146 |
+
subprocess.run(
|
| 147 |
+
["ffmpeg", "-y", "-i", str(video_path), "-vf", f"fps={fps}", "-q:v", "2",
|
| 148 |
+
str(tmp / "frame_%05d.jpg")],
|
| 149 |
+
capture_output=True, check=True, timeout=300,
|
| 150 |
+
)
|
| 151 |
+
frames = sorted(tmp.glob("frame_*.jpg"))
|
| 152 |
+
return [(i / fps, p) for i, p in enumerate(frames)]
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@torch.inference_mode()
|
| 156 |
+
def tag_frame(img_path: Path, threshold: float) -> list[str]:
|
| 157 |
+
img = Image.open(img_path).convert("RGB")
|
| 158 |
+
tensor = transform(img).unsqueeze(0).to(device)
|
| 159 |
+
model.threshold = threshold
|
| 160 |
+
tags, _ = model.generate_tag(tensor)
|
| 161 |
+
return [t.strip() for t in tags[0].split("|") if t.strip()]
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
| 165 |
+
|
| 166 |
+
def cache_path(video_name: str, fps: float, threshold: float) -> Path:
|
| 167 |
+
return CACHE_DIR / f"{Path(video_name).stem}_{fps}fps_t{threshold:.2f}.json"
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
@st.cache_data(show_spinner=False)
|
| 171 |
+
def get_tags(video_name: str, fps: float, threshold: float, video_dir: str) -> list[dict]:
|
| 172 |
+
cp = cache_path(video_name, fps, threshold)
|
| 173 |
+
if cp.exists():
|
| 174 |
+
return json.loads(cp.read_text())
|
| 175 |
+
|
| 176 |
+
frames = extract_frames(Path(video_dir) / video_name, fps)
|
| 177 |
+
results = []
|
| 178 |
+
bar = st.progress(0, text=f"Tagging {len(frames)} frames…")
|
| 179 |
+
for i, (ts, fp) in enumerate(frames):
|
| 180 |
+
results.append({"timestamp": round(ts, 2), "tags": tag_frame(fp, threshold), "frame": str(fp)})
|
| 181 |
+
bar.progress((i + 1) / len(frames), text=f"Frame {i+1}/{len(frames)}")
|
| 182 |
+
bar.empty()
|
| 183 |
+
|
| 184 |
+
cp.write_text(json.dumps(results, ensure_ascii=False))
|
| 185 |
+
return results
|
| 186 |
+
|
| 187 |
+
# ---------------------------------------------------------------------------
|
| 188 |
+
# UI
|
| 189 |
+
# ---------------------------------------------------------------------------
|
| 190 |
+
|
| 191 |
+
with st.sidebar:
|
| 192 |
+
st.header("Settings")
|
| 193 |
+
video_dir_input = st.text_input("Video directory", value=str(VIDEO_DIR))
|
| 194 |
+
video_dir = Path(video_dir_input)
|
| 195 |
+
|
| 196 |
+
videos = sorted(p.name for p in video_dir.glob("*.mp4")) if video_dir.exists() else []
|
| 197 |
+
if not videos:
|
| 198 |
+
st.error(f"No .mp4 files found in `{video_dir}`")
|
| 199 |
+
st.stop()
|
| 200 |
+
|
| 201 |
+
with st.sidebar:
|
| 202 |
+
video_name = st.selectbox(f"Video ({len(videos)} total)", videos)
|
| 203 |
+
|
| 204 |
+
col_search, col_fps, col_thresh = st.columns([4, 1, 1])
|
| 205 |
+
with col_search:
|
| 206 |
+
tag_filter = st.text_input("Search tags", placeholder="e.g. person, shelf, phone")
|
| 207 |
+
with col_fps:
|
| 208 |
+
fps = st.select_slider("Sampling fps", [0.25, 0.5, 1.0, 2.0], value=0.5)
|
| 209 |
+
with col_thresh:
|
| 210 |
+
threshold = st.slider("RAM Threshold", 0.1, 0.95, 0.68, step=0.05)
|
| 211 |
+
|
| 212 |
+
cached = cache_path(video_name, fps, threshold).exists()
|
| 213 |
+
st.caption(f"{'✓ cached' if cached else '⚡ will run inference'}")
|
| 214 |
+
|
| 215 |
+
with st.spinner(f"Processing {video_name}…"):
|
| 216 |
+
frames = get_tags(video_name, fps, threshold, str(video_dir))
|
| 217 |
+
|
| 218 |
+
# Filter by tag
|
| 219 |
+
if tag_filter.strip():
|
| 220 |
+
needles = [n.strip().lower() for n in tag_filter.split(",") if n.strip()]
|
| 221 |
+
frames = [f for f in frames if any(
|
| 222 |
+
any(n in t.lower() for t in f["tags"]) for n in needles
|
| 223 |
+
)]
|
| 224 |
+
st.info(f"{len(frames)} frames match **{tag_filter}**")
|
| 225 |
+
else:
|
| 226 |
+
# Show tag cloud when no filter active
|
| 227 |
+
all_tags: dict[str, int] = {}
|
| 228 |
+
for f in frames:
|
| 229 |
+
for t in f["tags"]:
|
| 230 |
+
all_tags[t] = all_tags.get(t, 0) + 1
|
| 231 |
+
top = sorted(all_tags.items(), key=lambda x: -x[1])[:60]
|
| 232 |
+
st.write("**All tags:** " + " · ".join(f"`{t}` ({n})" for t, n in top))
|
| 233 |
+
st.divider()
|
| 234 |
+
|
| 235 |
+
# Frame grid
|
| 236 |
+
if frames:
|
| 237 |
+
cols = st.columns(4)
|
| 238 |
+
for i, frame in enumerate(frames):
|
| 239 |
+
with cols[i % 4]:
|
| 240 |
+
st.image(frame["frame"], use_container_width=True)
|
| 241 |
+
ts = frame["timestamp"]
|
| 242 |
+
m, s = divmod(int(ts), 60)
|
| 243 |
+
st.caption(f"**{m}:{s:02d}** {' · '.join(frame['tags'][:5])}")
|
| 244 |
+
else:
|
| 245 |
+
st.warning("No frames match the search.")
|
| 246 |
+
|
| 247 |
+
```
|