ma7583 commited on
Commit
59bf898
·
verified ·
1 Parent(s): d85d60b

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +244 -0
README.md CHANGED
@@ -1,3 +1,247 @@
1
  ---
2
  license: mit
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  license: mit
3
  ---
4
+
5
+ ## setup uv env
6
+
7
+ ```
8
+ #!/bin/bash
9
+ # Set up RAM+ in an isolated venv with a compatible transformers version.
10
+ # recognize-anything uses APIs removed in transformers>=4.46, so we pin to 4.45.2.
11
+ set -e
12
+
13
+ VENV=".venv-ram"
14
+ WEIGHTS_DIR="models/ram_plus"
15
+ WEIGHTS_FILE="$WEIGHTS_DIR/ram_plus_swin_large_14m.pth"
16
+
17
+ # ---------------------------------------------------------------------------
18
+ # 1. Create venv
19
+ # ---------------------------------------------------------------------------
20
+ if [ ! -d "$VENV" ]; then
21
+ echo "Creating $VENV …"
22
+ uv venv "$VENV" --python 3.11
23
+ else
24
+ echo "$VENV already exists"
25
+ fi
26
+
27
+ PIP="$VENV/bin/pip"
28
+
29
+ # ---------------------------------------------------------------------------
30
+ # 2. Install PyTorch (reuse system CUDA version if available)
31
+ # ---------------------------------------------------------------------------
32
+ if ! "$VENV/bin/python" -c "import torch" 2>/dev/null; then
33
+ echo "Installing PyTorch …"
34
+ CUDA_VER=$(python3 -c "import torch; print('cu' + torch.version.cuda.replace('.',''))" 2>/dev/null || echo "cpu")
35
+ if [ "$CUDA_VER" = "cpu" ]; then
36
+ uv pip install --python "$VENV/bin/python" torch torchvision
37
+ else
38
+ uv pip install --python "$VENV/bin/python" torch torchvision \
39
+ --index-url "https://download.pytorch.org/whl/$CUDA_VER"
40
+ fi
41
+ fi
42
+
43
+ # ---------------------------------------------------------------------------
44
+ # 3. Install recognize-anything + pinned transformers
45
+ # ---------------------------------------------------------------------------
46
+ if ! "$VENV/bin/python" -c "import ram" 2>/dev/null; then
47
+ echo "Installing recognize-anything with transformers==4.45.2 …"
48
+ uv pip install --python "$VENV/bin/python" \
49
+ "transformers==4.45.2" \
50
+ timm pillow scipy fairscale \
51
+ git+https://github.com/xinyu1205/recognize-anything.git
52
+ fi
53
+
54
+ # ---------------------------------------------------------------------------
55
+ # 4. Install Streamlit in the RAM venv
56
+ # ---------------------------------------------------------------------------
57
+ if ! "$VENV/bin/python" -c "import streamlit" 2>/dev/null; then
58
+ echo "Installing streamlit …"
59
+ uv pip install --python "$VENV/bin/python" streamlit
60
+ fi
61
+
62
+ # ---------------------------------------------------------------------------
63
+ # 5. Download weights
64
+ # ---------------------------------------------------------------------------
65
+ mkdir -p "$WEIGHTS_DIR"
66
+ if [ ! -f "$WEIGHTS_FILE" ]; then
67
+ echo "Downloading RAM+ weights (~3 GB) …"
68
+ "$VENV/bin/python" -c "
69
+ from huggingface_hub import hf_hub_download
70
+ path = hf_hub_download(
71
+ repo_id='ma7583/ramplus',
72
+ filename='ram_plus_swin_large_14m.pth',
73
+ local_dir='$WEIGHTS_DIR',
74
+ )
75
+ print('Downloaded to:', path)
76
+ "
77
+ else
78
+ echo "Weights already present at $WEIGHTS_FILE"
79
+ fi
80
+
81
+ echo ""
82
+ echo "Setup complete. Run the app with:"
83
+ echo " $VENV/bin/streamlit run tests/explore_ram.py"
84
+
85
+
86
+ ```
87
+
88
+
89
+ ## run streamlit for demo
90
+
91
+ ```
92
+ """RAM+ Video Object Explorer — Streamlit app.
93
+
94
+ Usage:
95
+ streamlit run tests/explore_ram.py
96
+ """
97
+
98
+ import json
99
+ import subprocess
100
+ import tempfile
101
+ from pathlib import Path
102
+ from PIL import Image
103
+ import streamlit as st
104
+ import torch
105
+
106
+ VIDEO_DIR = Path(".")
107
+ WEIGHTS = Path("models/ram_plus/ram_plus_swin_large_14m.pth")
108
+ CACHE_DIR = Path("tests/mbs/ram_tags")
109
+ IMAGE_SIZE = 384
110
+
111
+ st.set_page_config(page_title="RAM+ Explorer", layout="wide")
112
+
113
+ # ---------------------------------------------------------------------------
114
+ # Load model
115
+ # ---------------------------------------------------------------------------
116
+
117
+ @st.cache_resource(show_spinner="Loading RAM+ model…")
118
+ def load_model(weights: str):
119
+ from ram.models import ram_plus
120
+ import torchvision.transforms as T
121
+
122
+ device = "cuda" if torch.cuda.is_available() else "cpu"
123
+ model = ram_plus(pretrained=weights, image_size=IMAGE_SIZE, vit="swin_l")
124
+ model.eval().to(device)
125
+
126
+ transform = T.Compose([
127
+ T.Resize((IMAGE_SIZE, IMAGE_SIZE)),
128
+ T.ToTensor(),
129
+ T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
130
+ ])
131
+ return model, transform, device
132
+
133
+
134
+ if not WEIGHTS.exists():
135
+ st.error(f"Weights not found at `{WEIGHTS}`. Run `bash run_ram_setup.sh` first.")
136
+ st.stop()
137
+
138
+ model, transform, device = load_model(str(WEIGHTS))
139
+
140
+ # ---------------------------------------------------------------------------
141
+ # Helpers
142
+ # ---------------------------------------------------------------------------
143
+
144
+ def extract_frames(video_path: Path, fps: float) -> list[tuple[float, Path]]:
145
+ tmp = Path(tempfile.mkdtemp(prefix="ram-frames-"))
146
+ subprocess.run(
147
+ ["ffmpeg", "-y", "-i", str(video_path), "-vf", f"fps={fps}", "-q:v", "2",
148
+ str(tmp / "frame_%05d.jpg")],
149
+ capture_output=True, check=True, timeout=300,
150
+ )
151
+ frames = sorted(tmp.glob("frame_*.jpg"))
152
+ return [(i / fps, p) for i, p in enumerate(frames)]
153
+
154
+
155
+ @torch.inference_mode()
156
+ def tag_frame(img_path: Path, threshold: float) -> list[str]:
157
+ img = Image.open(img_path).convert("RGB")
158
+ tensor = transform(img).unsqueeze(0).to(device)
159
+ model.threshold = threshold
160
+ tags, _ = model.generate_tag(tensor)
161
+ return [t.strip() for t in tags[0].split("|") if t.strip()]
162
+
163
+
164
+ CACHE_DIR.mkdir(parents=True, exist_ok=True)
165
+
166
+ def cache_path(video_name: str, fps: float, threshold: float) -> Path:
167
+ return CACHE_DIR / f"{Path(video_name).stem}_{fps}fps_t{threshold:.2f}.json"
168
+
169
+
170
+ @st.cache_data(show_spinner=False)
171
+ def get_tags(video_name: str, fps: float, threshold: float, video_dir: str) -> list[dict]:
172
+ cp = cache_path(video_name, fps, threshold)
173
+ if cp.exists():
174
+ return json.loads(cp.read_text())
175
+
176
+ frames = extract_frames(Path(video_dir) / video_name, fps)
177
+ results = []
178
+ bar = st.progress(0, text=f"Tagging {len(frames)} frames…")
179
+ for i, (ts, fp) in enumerate(frames):
180
+ results.append({"timestamp": round(ts, 2), "tags": tag_frame(fp, threshold), "frame": str(fp)})
181
+ bar.progress((i + 1) / len(frames), text=f"Frame {i+1}/{len(frames)}")
182
+ bar.empty()
183
+
184
+ cp.write_text(json.dumps(results, ensure_ascii=False))
185
+ return results
186
+
187
+ # ---------------------------------------------------------------------------
188
+ # UI
189
+ # ---------------------------------------------------------------------------
190
+
191
+ with st.sidebar:
192
+ st.header("Settings")
193
+ video_dir_input = st.text_input("Video directory", value=str(VIDEO_DIR))
194
+ video_dir = Path(video_dir_input)
195
+
196
+ videos = sorted(p.name for p in video_dir.glob("*.mp4")) if video_dir.exists() else []
197
+ if not videos:
198
+ st.error(f"No .mp4 files found in `{video_dir}`")
199
+ st.stop()
200
+
201
+ with st.sidebar:
202
+ video_name = st.selectbox(f"Video ({len(videos)} total)", videos)
203
+
204
+ col_search, col_fps, col_thresh = st.columns([4, 1, 1])
205
+ with col_search:
206
+ tag_filter = st.text_input("Search tags", placeholder="e.g. person, shelf, phone")
207
+ with col_fps:
208
+ fps = st.select_slider("Sampling fps", [0.25, 0.5, 1.0, 2.0], value=0.5)
209
+ with col_thresh:
210
+ threshold = st.slider("RAM Threshold", 0.1, 0.95, 0.68, step=0.05)
211
+
212
+ cached = cache_path(video_name, fps, threshold).exists()
213
+ st.caption(f"{'✓ cached' if cached else '⚡ will run inference'}")
214
+
215
+ with st.spinner(f"Processing {video_name}…"):
216
+ frames = get_tags(video_name, fps, threshold, str(video_dir))
217
+
218
+ # Filter by tag
219
+ if tag_filter.strip():
220
+ needles = [n.strip().lower() for n in tag_filter.split(",") if n.strip()]
221
+ frames = [f for f in frames if any(
222
+ any(n in t.lower() for t in f["tags"]) for n in needles
223
+ )]
224
+ st.info(f"{len(frames)} frames match **{tag_filter}**")
225
+ else:
226
+ # Show tag cloud when no filter active
227
+ all_tags: dict[str, int] = {}
228
+ for f in frames:
229
+ for t in f["tags"]:
230
+ all_tags[t] = all_tags.get(t, 0) + 1
231
+ top = sorted(all_tags.items(), key=lambda x: -x[1])[:60]
232
+ st.write("**All tags:** " + " · ".join(f"`{t}` ({n})" for t, n in top))
233
+ st.divider()
234
+
235
+ # Frame grid
236
+ if frames:
237
+ cols = st.columns(4)
238
+ for i, frame in enumerate(frames):
239
+ with cols[i % 4]:
240
+ st.image(frame["frame"], use_container_width=True)
241
+ ts = frame["timestamp"]
242
+ m, s = divmod(int(ts), 60)
243
+ st.caption(f"**{m}:{s:02d}** {' · '.join(frame['tags'][:5])}")
244
+ else:
245
+ st.warning("No frames match the search.")
246
+
247
+ ```