multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
43cc06a verified
Raw History Blame Contribute Delete
14.4 kB
"""Oasis scene parser: photo -> structured JSON scene record (objects, boxes, masks, OCR text, relations).
Oasis is a 2-bit CPU engine (one C++17 file). We download the engine source + weights from the model
repo, compile it once at startup with the Space's own compiler (-march=native matches the host CPU),
and call it through the authors' ctypes binding.
"""
import json
import math
import os
import sys
import threading
import time
def _effective_cpus() -> int:
"""CPUs we may actually use: cgroup quota (containers report the host's core count) and affinity."""
n = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)
try:
quota, period = open("/sys/fs/cgroup/cpu.max").read().split()[:2]
if quota != "max":
n = min(n, max(1, math.floor(int(quota) / int(period))))
except Exception:
try:
q = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read())
p = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read())
if q > 0:
n = min(n, max(1, q // p))
except Exception:
pass
return n
# The engine spins its OpenMP workers; oversubscribing a CPU-limited container makes it ~15x slower.
os.environ.setdefault("OMP_NUM_THREADS", str(_effective_cpus()))
import gradio as gr
from huggingface_hub import snapshot_download
from PIL import Image, ImageDraw, ImageFont
MODEL_ID = "phanerozoic/oasis"
LANGUAGES = ["en", "de", "fr", "es", "it", "nl", "pl"]
# ---------------------------------------------------------------- engine setup
repo_dir = snapshot_download(MODEL_ID, allow_patterns=["oasis.py", "oasis.cpp", "oasis.oasis"])
build_dir = "/tmp/oasis_build"
os.makedirs(build_dir, exist_ok=True)
sys.path.insert(0, repo_dir)
import oasis as oasis_mod # noqa: E402 (authors' binding)
lib_path = os.path.join(build_dir, oasis_mod.LIBRARY)
if not os.path.isfile(lib_path):
t0 = time.time()
oasis_mod.build(out=lib_path)
print(f"[oasis] engine compiled in {time.time() - t0:.1f}s -> {lib_path}", flush=True)
MODEL = oasis_mod.Oasis(os.path.join(repo_dir, "oasis.oasis"), library=lib_path)
LOCK = threading.Lock() # one engine handle with a shared output buffer
print(f"[oasis] engine loaded, OMP_NUM_THREADS={os.environ['OMP_NUM_THREADS']} (os.cpu_count={os.cpu_count()})", flush=True)
# ---------------------------------------------------------------- drawing
PALETTE = [
(230, 25, 75), (60, 180, 75), (0, 130, 200), (245, 130, 48), (145, 30, 180),
(70, 240, 240), (240, 50, 230), (210, 245, 60), (0, 128, 128), (170, 110, 40),
]
def _scale(pts, w, h):
return [(pts[i] * w / 1000.0, pts[i + 1] * h / 1000.0) for i in range(0, len(pts) - 1, 2)]
def _font(size):
try:
return ImageFont.truetype("DejaVuSans-Bold.ttf", size)
except Exception:
return ImageFont.load_default()
def draw_record(image: Image.Image, record: dict) -> Image.Image:
img = image.convert("RGB")
w, h = img.size
overlay = Image.new("RGBA", img.size, (0, 0, 0, 0))
od = ImageDraw.Draw(overlay)
lw = max(2, round(min(w, h) / 300))
fs = max(12, round(min(w, h) / 45))
font = _font(fs)
objects = record.get("objects") or []
for i, obj in enumerate(objects):
c = PALETTE[i % len(PALETTE)]
for poly in obj.get("mask") or []:
if isinstance(poly, list) and len(poly) >= 6 and all(isinstance(v, (int, float)) for v in poly):
od.polygon(_scale(poly, w, h), fill=c + (80,), outline=c + (220,))
img = Image.alpha_composite(img.convert("RGBA"), overlay).convert("RGB")
d = ImageDraw.Draw(img)
def label(x0, y0, text, c):
tb = d.textbbox((x0, y0), text, font=font)
y = y0 - (tb[3] - tb[1]) - 6 if y0 - (tb[3] - tb[1]) - 6 >= 0 else y0
tb = d.textbbox((x0 + 3, y + 2), text, font=font)
d.rectangle([tb[0] - 3, tb[1] - 2, tb[2] + 3, tb[3] + 2], fill=c)
d.text((x0 + 3, y + 2), text, fill=(255, 255, 255), font=font)
for i, obj in enumerate(objects):
b = obj.get("bbox")
if not b or len(b) != 4:
continue
c = PALETTE[i % len(PALETTE)]
x0, y0, x1, y1 = b[0] * w / 1000, b[1] * h / 1000, b[2] * w / 1000, b[3] * h / 1000
d.rectangle([x0, y0, x1, y1], outline=c, width=lw)
score = obj.get("score")
label(x0, y0, f"{obj.get('id', i + 1)} {obj.get('name', '?')}" + (f" {score:.2f}" if isinstance(score, (int, float)) else ""), c)
for t in record.get("text") or []:
b = t.get("bbox")
if not b or len(b) != 4:
continue
c = (255, 200, 0)
x0, y0, x1, y1 = b[0] * w / 1000, b[1] * h / 1000, b[2] * w / 1000, b[3] * h / 1000
d.rectangle([x0, y0, x1, y1], outline=c, width=lw)
label(x1 + 4, y0 + fs + 6, f"“{t.get('string', '')}”", (40, 40, 40))
# form answers that are boxes
form = record.get("form") or {}
for i, (name, field) in enumerate(form.items()):
v = field.get("value") if isinstance(field, dict) else None
if isinstance(v, list) and len(v) == 4 and all(isinstance(x, (int, float)) for x in v):
c = PALETTE[(i + 3) % len(PALETTE)]
x0, y0, x1, y1 = v[0] * w / 1000, v[1] * h / 1000, v[2] * w / 1000, v[3] * h / 1000
d.rectangle([x0, y0, x1, y1], outline=c, width=lw + 1)
label(x0, y0, name, c)
return img
def _load(image) -> Image.Image:
if image is None:
raise gr.Error("Please upload an image.")
if isinstance(image, str):
image = Image.open(image)
from PIL import ImageOps
return ImageOps.exif_transpose(image).convert("RGB")
def _run(img: Image.Image, options: dict) -> dict:
with LOCK:
try:
return MODEL.describe(img, **options)
except RuntimeError as e:
raise gr.Error(f"Oasis engine error: {e}")
# ---------------------------------------------------------------- handlers
def describe(
image,
language: str = "en",
keywords: str = "",
conf: float = 0.3,
mask_format: str = "polygons",
reasoning: bool = True,
max_objects: int = 20,
):
"""Describe a photo as a structured JSON scene record.
Args:
image: the input photo (filepath).
language: description language, one of en, de, fr, es, it, nl, pl.
keywords: optional comma-separated words to steer the description toward.
conf: object confidence threshold (0-1).
mask_format: "polygons", "largest", "merged" or "none".
reasoning: include the model's short reasoning trace.
max_objects: maximum number of objects to report.
Returns:
An annotated image (boxes, masks, read text) and the JSON scene record.
Boxes and outlines use a 0-1000 scale on each image axis.
"""
img = _load(image)
options = {
"language": language,
"conf": float(conf),
"max_objects": int(max_objects),
"reasoning": bool(reasoning),
}
if mask_format == "none":
options["masks"] = False
else:
options["mask_format"] = mask_format
kw = [k.strip() for k in (keywords or "").split(",") if k.strip()]
if kw:
options["keywords"] = kw
record = _run(img, options)
return draw_record(img, record), record
DEFAULT_FORM = json.dumps(
{
"vehicle": {"enum": ["motorcycle", "bicycle", "car", "truck"]},
"color": {"enum": ["red", "blue", "black", "white"]},
"parked_cars": {"type": "integer", "minimum": 0, "maximum": 10},
"where": {"x-source": "bbox", "x-of": "vehicle"},
},
indent=2,
)
def fill_form(image, form_json: str = DEFAULT_FORM):
"""Fill an ordered form about a photo.
Fields: {"name": {"enum": [...]}} for a choice, {"type": "integer", "minimum": 0, "maximum": 9} for a count,
{"type": "string"} for free text, and {"x-source": "bbox", "x-of": "<earlier choice field>"} for a box.
Args:
image: the input photo (filepath).
form_json: the form schema as JSON text.
Returns:
An annotated image (box answers drawn) and the filled form with per-answer probabilities.
"""
img = _load(image)
try:
form = json.loads(form_json)
except json.JSONDecodeError as e:
raise gr.Error(f"Form is not valid JSON: {e}")
if not isinstance(form, dict) or not form:
raise gr.Error("Form must be a non-empty JSON object of fields.")
record = _run(img, {"form": form})
return draw_record(img, record), record
# ---------------------------------------------------------------- UI
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks(title="Oasis scene parser") as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"# 🏝️ Oasis — photo → structured scene JSON\n"
"A 19 MB, 2-bit vision model (Cactus Compute's Mirage with a new CPU engine and a retrained text reader) "
"that turns a photo into a JSON record: objects with boxes and masks, text it can read, relations and a "
"short reasoning trace — or fills in a form you define. Runs on CPU. "
"[Model](https://huggingface.co/phanerozoic/oasis) · [Base model: Mirage](https://huggingface.co/Cactus-Compute/mirage)"
)
with gr.Tabs():
with gr.Tab("Describe a photo"):
with gr.Row():
with gr.Column():
d_image = gr.Image(type="filepath", label="Photo", height=420)
d_btn = gr.Button("Describe", variant="primary")
with gr.Accordion("Options", open=False):
d_lang = gr.Dropdown(LANGUAGES, value="en", label="Description language")
d_kw = gr.Textbox(label="Keywords (comma-separated, optional)", placeholder="e.g. sign, car")
d_conf = gr.Slider(0.05, 0.95, value=0.3, step=0.05, label="Object confidence threshold")
d_mask = gr.Radio(["polygons", "largest", "merged", "none"], value="polygons", label="Mask format")
d_reason = gr.Checkbox(value=True, label="Include reasoning trace")
d_maxobj = gr.Slider(1, 64, value=20, step=1, label="Max objects")
with gr.Column():
d_out_img = gr.Image(label="Annotated", height=420)
d_out_json = gr.JSON(label="Scene record")
gr.Examples(
examples=[
["examples/sign_beach.jpg"],
["examples/motorcycle_street.jpg"],
["examples/girl_with_dog.jpg"],
["examples/cafe_interior.jpg"],
["examples/living_room_brick.jpg"],
],
inputs=[d_image],
outputs=[d_out_img, d_out_json],
fn=describe,
cache_examples=True,
cache_mode="lazy",
)
d_btn.click(
describe,
inputs=[d_image, d_lang, d_kw, d_conf, d_mask, d_reason, d_maxobj],
outputs=[d_out_img, d_out_json],
api_name="describe",
)
with gr.Tab("Fill a form"):
with gr.Row():
with gr.Column():
f_image = gr.Image(type="filepath", label="Photo", height=420)
f_form = gr.Code(value=DEFAULT_FORM, language="json", label="Form (ordered fields)")
f_btn = gr.Button("Fill form", variant="primary")
gr.Markdown(
"Field types: `{\"enum\": [...]}` choice · `{\"type\": \"integer\", \"minimum\": 0, \"maximum\": 9}` count · "
"`{\"type\": \"string\"}` text · `{\"x-source\": \"bbox\", \"x-of\": \"<earlier choice field>\"}` box."
)
with gr.Column():
f_out_img = gr.Image(label="Annotated", height=420)
f_out_json = gr.JSON(label="Filled form")
gr.Examples(
examples=[
["examples/motorcycle_street.jpg", DEFAULT_FORM],
[
"examples/sign_beach.jpg",
json.dumps(
{
"sign": {"enum": ["stop sign", "letter board", "billboard"]},
"place": {"enum": ["beach", "city", "forest", "kitchen"]},
"cars": {"type": "integer", "minimum": 0, "maximum": 10},
"where": {"x-source": "bbox", "x-of": "sign"},
},
indent=2,
),
],
[
"examples/girl_with_dog.jpg",
json.dumps(
{
"animal": {"enum": ["dog", "cat", "horse", "bird"]},
"animals": {"type": "integer", "minimum": 0, "maximum": 10},
"people": {"type": "integer", "minimum": 0, "maximum": 10},
"where": {"x-source": "bbox", "x-of": "animal"},
},
indent=2,
),
],
],
inputs=[f_image, f_form],
outputs=[f_out_img, f_out_json],
fn=fill_form,
cache_examples=True,
cache_mode="lazy",
)
f_btn.click(
fill_form,
inputs=[f_image, f_form],
outputs=[f_out_img, f_out_json],
api_name="fill_form",
)
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)