Spaces:
Running
Running
Download app.py from hugging-apps/oasis-scene-parser: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/spaces/hugging-apps/oasis-scene-parser/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/oasis-scene-parser/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/oasis-scene-parser/resolve/main/app.py
14.4 kB
| """Oasis scene parser: photo -> structured JSON scene record (objects, boxes, masks, OCR text, relations). | |
| Oasis is a 2-bit CPU engine (one C++17 file). We download the engine source + weights from the model | |
| repo, compile it once at startup with the Space's own compiler (-march=native matches the host CPU), | |
| and call it through the authors' ctypes binding. | |
| """ | |
| import json | |
| import math | |
| import os | |
| import sys | |
| import threading | |
| import time | |
| def _effective_cpus() -> int: | |
| """CPUs we may actually use: cgroup quota (containers report the host's core count) and affinity.""" | |
| n = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1) | |
| try: | |
| quota, period = open("/sys/fs/cgroup/cpu.max").read().split()[:2] | |
| if quota != "max": | |
| n = min(n, max(1, math.floor(int(quota) / int(period)))) | |
| except Exception: | |
| try: | |
| q = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read()) | |
| p = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read()) | |
| if q > 0: | |
| n = min(n, max(1, q // p)) | |
| except Exception: | |
| pass | |
| return n | |
| # The engine spins its OpenMP workers; oversubscribing a CPU-limited container makes it ~15x slower. | |
| os.environ.setdefault("OMP_NUM_THREADS", str(_effective_cpus())) | |
| import gradio as gr | |
| from huggingface_hub import snapshot_download | |
| from PIL import Image, ImageDraw, ImageFont | |
| MODEL_ID = "phanerozoic/oasis" | |
| LANGUAGES = ["en", "de", "fr", "es", "it", "nl", "pl"] | |
| # ---------------------------------------------------------------- engine setup | |
| repo_dir = snapshot_download(MODEL_ID, allow_patterns=["oasis.py", "oasis.cpp", "oasis.oasis"]) | |
| build_dir = "/tmp/oasis_build" | |
| os.makedirs(build_dir, exist_ok=True) | |
| sys.path.insert(0, repo_dir) | |
| import oasis as oasis_mod # noqa: E402 (authors' binding) | |
| lib_path = os.path.join(build_dir, oasis_mod.LIBRARY) | |
| if not os.path.isfile(lib_path): | |
| t0 = time.time() | |
| oasis_mod.build(out=lib_path) | |
| print(f"[oasis] engine compiled in {time.time() - t0:.1f}s -> {lib_path}", flush=True) | |
| MODEL = oasis_mod.Oasis(os.path.join(repo_dir, "oasis.oasis"), library=lib_path) | |
| LOCK = threading.Lock() # one engine handle with a shared output buffer | |
| print(f"[oasis] engine loaded, OMP_NUM_THREADS={os.environ['OMP_NUM_THREADS']} (os.cpu_count={os.cpu_count()})", flush=True) | |
| # ---------------------------------------------------------------- drawing | |
| PALETTE = [ | |
| (230, 25, 75), (60, 180, 75), (0, 130, 200), (245, 130, 48), (145, 30, 180), | |
| (70, 240, 240), (240, 50, 230), (210, 245, 60), (0, 128, 128), (170, 110, 40), | |
| ] | |
| def _scale(pts, w, h): | |
| return [(pts[i] * w / 1000.0, pts[i + 1] * h / 1000.0) for i in range(0, len(pts) - 1, 2)] | |
| def _font(size): | |
| try: | |
| return ImageFont.truetype("DejaVuSans-Bold.ttf", size) | |
| except Exception: | |
| return ImageFont.load_default() | |
| def draw_record(image: Image.Image, record: dict) -> Image.Image: | |
| img = image.convert("RGB") | |
| w, h = img.size | |
| overlay = Image.new("RGBA", img.size, (0, 0, 0, 0)) | |
| od = ImageDraw.Draw(overlay) | |
| lw = max(2, round(min(w, h) / 300)) | |
| fs = max(12, round(min(w, h) / 45)) | |
| font = _font(fs) | |
| objects = record.get("objects") or [] | |
| for i, obj in enumerate(objects): | |
| c = PALETTE[i % len(PALETTE)] | |
| for poly in obj.get("mask") or []: | |
| if isinstance(poly, list) and len(poly) >= 6 and all(isinstance(v, (int, float)) for v in poly): | |
| od.polygon(_scale(poly, w, h), fill=c + (80,), outline=c + (220,)) | |
| img = Image.alpha_composite(img.convert("RGBA"), overlay).convert("RGB") | |
| d = ImageDraw.Draw(img) | |
| def label(x0, y0, text, c): | |
| tb = d.textbbox((x0, y0), text, font=font) | |
| y = y0 - (tb[3] - tb[1]) - 6 if y0 - (tb[3] - tb[1]) - 6 >= 0 else y0 | |
| tb = d.textbbox((x0 + 3, y + 2), text, font=font) | |
| d.rectangle([tb[0] - 3, tb[1] - 2, tb[2] + 3, tb[3] + 2], fill=c) | |
| d.text((x0 + 3, y + 2), text, fill=(255, 255, 255), font=font) | |
| for i, obj in enumerate(objects): | |
| b = obj.get("bbox") | |
| if not b or len(b) != 4: | |
| continue | |
| c = PALETTE[i % len(PALETTE)] | |
| x0, y0, x1, y1 = b[0] * w / 1000, b[1] * h / 1000, b[2] * w / 1000, b[3] * h / 1000 | |
| d.rectangle([x0, y0, x1, y1], outline=c, width=lw) | |
| score = obj.get("score") | |
| label(x0, y0, f"{obj.get('id', i + 1)} {obj.get('name', '?')}" + (f" {score:.2f}" if isinstance(score, (int, float)) else ""), c) | |
| for t in record.get("text") or []: | |
| b = t.get("bbox") | |
| if not b or len(b) != 4: | |
| continue | |
| c = (255, 200, 0) | |
| x0, y0, x1, y1 = b[0] * w / 1000, b[1] * h / 1000, b[2] * w / 1000, b[3] * h / 1000 | |
| d.rectangle([x0, y0, x1, y1], outline=c, width=lw) | |
| label(x1 + 4, y0 + fs + 6, f"“{t.get('string', '')}”", (40, 40, 40)) | |
| # form answers that are boxes | |
| form = record.get("form") or {} | |
| for i, (name, field) in enumerate(form.items()): | |
| v = field.get("value") if isinstance(field, dict) else None | |
| if isinstance(v, list) and len(v) == 4 and all(isinstance(x, (int, float)) for x in v): | |
| c = PALETTE[(i + 3) % len(PALETTE)] | |
| x0, y0, x1, y1 = v[0] * w / 1000, v[1] * h / 1000, v[2] * w / 1000, v[3] * h / 1000 | |
| d.rectangle([x0, y0, x1, y1], outline=c, width=lw + 1) | |
| label(x0, y0, name, c) | |
| return img | |
| def _load(image) -> Image.Image: | |
| if image is None: | |
| raise gr.Error("Please upload an image.") | |
| if isinstance(image, str): | |
| image = Image.open(image) | |
| from PIL import ImageOps | |
| return ImageOps.exif_transpose(image).convert("RGB") | |
| def _run(img: Image.Image, options: dict) -> dict: | |
| with LOCK: | |
| try: | |
| return MODEL.describe(img, **options) | |
| except RuntimeError as e: | |
| raise gr.Error(f"Oasis engine error: {e}") | |
| # ---------------------------------------------------------------- handlers | |
| def describe( | |
| image, | |
| language: str = "en", | |
| keywords: str = "", | |
| conf: float = 0.3, | |
| mask_format: str = "polygons", | |
| reasoning: bool = True, | |
| max_objects: int = 20, | |
| ): | |
| """Describe a photo as a structured JSON scene record. | |
| Args: | |
| image: the input photo (filepath). | |
| language: description language, one of en, de, fr, es, it, nl, pl. | |
| keywords: optional comma-separated words to steer the description toward. | |
| conf: object confidence threshold (0-1). | |
| mask_format: "polygons", "largest", "merged" or "none". | |
| reasoning: include the model's short reasoning trace. | |
| max_objects: maximum number of objects to report. | |
| Returns: | |
| An annotated image (boxes, masks, read text) and the JSON scene record. | |
| Boxes and outlines use a 0-1000 scale on each image axis. | |
| """ | |
| img = _load(image) | |
| options = { | |
| "language": language, | |
| "conf": float(conf), | |
| "max_objects": int(max_objects), | |
| "reasoning": bool(reasoning), | |
| } | |
| if mask_format == "none": | |
| options["masks"] = False | |
| else: | |
| options["mask_format"] = mask_format | |
| kw = [k.strip() for k in (keywords or "").split(",") if k.strip()] | |
| if kw: | |
| options["keywords"] = kw | |
| record = _run(img, options) | |
| return draw_record(img, record), record | |
| DEFAULT_FORM = json.dumps( | |
| { | |
| "vehicle": {"enum": ["motorcycle", "bicycle", "car", "truck"]}, | |
| "color": {"enum": ["red", "blue", "black", "white"]}, | |
| "parked_cars": {"type": "integer", "minimum": 0, "maximum": 10}, | |
| "where": {"x-source": "bbox", "x-of": "vehicle"}, | |
| }, | |
| indent=2, | |
| ) | |
| def fill_form(image, form_json: str = DEFAULT_FORM): | |
| """Fill an ordered form about a photo. | |
| Fields: {"name": {"enum": [...]}} for a choice, {"type": "integer", "minimum": 0, "maximum": 9} for a count, | |
| {"type": "string"} for free text, and {"x-source": "bbox", "x-of": "<earlier choice field>"} for a box. | |
| Args: | |
| image: the input photo (filepath). | |
| form_json: the form schema as JSON text. | |
| Returns: | |
| An annotated image (box answers drawn) and the filled form with per-answer probabilities. | |
| """ | |
| img = _load(image) | |
| try: | |
| form = json.loads(form_json) | |
| except json.JSONDecodeError as e: | |
| raise gr.Error(f"Form is not valid JSON: {e}") | |
| if not isinstance(form, dict) or not form: | |
| raise gr.Error("Form must be a non-empty JSON object of fields.") | |
| record = _run(img, {"form": form}) | |
| return draw_record(img, record), record | |
| # ---------------------------------------------------------------- UI | |
| CSS = """ | |
| #col-container { max-width: 1200px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks(title="Oasis scene parser") as demo: | |
| with gr.Column(elem_id="col-container"): | |
| gr.Markdown( | |
| "# 🏝️ Oasis — photo → structured scene JSON\n" | |
| "A 19 MB, 2-bit vision model (Cactus Compute's Mirage with a new CPU engine and a retrained text reader) " | |
| "that turns a photo into a JSON record: objects with boxes and masks, text it can read, relations and a " | |
| "short reasoning trace — or fills in a form you define. Runs on CPU. " | |
| "[Model](https://huggingface.co/phanerozoic/oasis) · [Base model: Mirage](https://huggingface.co/Cactus-Compute/mirage)" | |
| ) | |
| with gr.Tabs(): | |
| with gr.Tab("Describe a photo"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| d_image = gr.Image(type="filepath", label="Photo", height=420) | |
| d_btn = gr.Button("Describe", variant="primary") | |
| with gr.Accordion("Options", open=False): | |
| d_lang = gr.Dropdown(LANGUAGES, value="en", label="Description language") | |
| d_kw = gr.Textbox(label="Keywords (comma-separated, optional)", placeholder="e.g. sign, car") | |
| d_conf = gr.Slider(0.05, 0.95, value=0.3, step=0.05, label="Object confidence threshold") | |
| d_mask = gr.Radio(["polygons", "largest", "merged", "none"], value="polygons", label="Mask format") | |
| d_reason = gr.Checkbox(value=True, label="Include reasoning trace") | |
| d_maxobj = gr.Slider(1, 64, value=20, step=1, label="Max objects") | |
| with gr.Column(): | |
| d_out_img = gr.Image(label="Annotated", height=420) | |
| d_out_json = gr.JSON(label="Scene record") | |
| gr.Examples( | |
| examples=[ | |
| ["examples/sign_beach.jpg"], | |
| ["examples/motorcycle_street.jpg"], | |
| ["examples/girl_with_dog.jpg"], | |
| ["examples/cafe_interior.jpg"], | |
| ["examples/living_room_brick.jpg"], | |
| ], | |
| inputs=[d_image], | |
| outputs=[d_out_img, d_out_json], | |
| fn=describe, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| d_btn.click( | |
| describe, | |
| inputs=[d_image, d_lang, d_kw, d_conf, d_mask, d_reason, d_maxobj], | |
| outputs=[d_out_img, d_out_json], | |
| api_name="describe", | |
| ) | |
| with gr.Tab("Fill a form"): | |
| with gr.Row(): | |
| with gr.Column(): | |
| f_image = gr.Image(type="filepath", label="Photo", height=420) | |
| f_form = gr.Code(value=DEFAULT_FORM, language="json", label="Form (ordered fields)") | |
| f_btn = gr.Button("Fill form", variant="primary") | |
| gr.Markdown( | |
| "Field types: `{\"enum\": [...]}` choice · `{\"type\": \"integer\", \"minimum\": 0, \"maximum\": 9}` count · " | |
| "`{\"type\": \"string\"}` text · `{\"x-source\": \"bbox\", \"x-of\": \"<earlier choice field>\"}` box." | |
| ) | |
| with gr.Column(): | |
| f_out_img = gr.Image(label="Annotated", height=420) | |
| f_out_json = gr.JSON(label="Filled form") | |
| gr.Examples( | |
| examples=[ | |
| ["examples/motorcycle_street.jpg", DEFAULT_FORM], | |
| [ | |
| "examples/sign_beach.jpg", | |
| json.dumps( | |
| { | |
| "sign": {"enum": ["stop sign", "letter board", "billboard"]}, | |
| "place": {"enum": ["beach", "city", "forest", "kitchen"]}, | |
| "cars": {"type": "integer", "minimum": 0, "maximum": 10}, | |
| "where": {"x-source": "bbox", "x-of": "sign"}, | |
| }, | |
| indent=2, | |
| ), | |
| ], | |
| [ | |
| "examples/girl_with_dog.jpg", | |
| json.dumps( | |
| { | |
| "animal": {"enum": ["dog", "cat", "horse", "bird"]}, | |
| "animals": {"type": "integer", "minimum": 0, "maximum": 10}, | |
| "people": {"type": "integer", "minimum": 0, "maximum": 10}, | |
| "where": {"x-source": "bbox", "x-of": "animal"}, | |
| }, | |
| indent=2, | |
| ), | |
| ], | |
| ], | |
| inputs=[f_image, f_form], | |
| outputs=[f_out_img, f_out_json], | |
| fn=fill_form, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| f_btn.click( | |
| fill_form, | |
| inputs=[f_image, f_form], | |
| outputs=[f_out_img, f_out_json], | |
| api_name="fill_form", | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True) | |