"""Oasis scene parser: photo -> structured JSON scene record (objects, boxes, masks, OCR text, relations). Oasis is a 2-bit CPU engine (one C++17 file). We download the engine source + weights from the model repo, compile it once at startup with the Space's own compiler (-march=native matches the host CPU), and call it through the authors' ctypes binding. """ import json import math import os import sys import threading import time def _effective_cpus() -> int: """CPUs we may actually use: cgroup quota (containers report the host's core count) and affinity.""" n = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1) try: quota, period = open("/sys/fs/cgroup/cpu.max").read().split()[:2] if quota != "max": n = min(n, max(1, math.floor(int(quota) / int(period)))) except Exception: try: q = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read()) p = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read()) if q > 0: n = min(n, max(1, q // p)) except Exception: pass return n # The engine spins its OpenMP workers; oversubscribing a CPU-limited container makes it ~15x slower. os.environ.setdefault("OMP_NUM_THREADS", str(_effective_cpus())) import gradio as gr from huggingface_hub import snapshot_download from PIL import Image, ImageDraw, ImageFont MODEL_ID = "phanerozoic/oasis" LANGUAGES = ["en", "de", "fr", "es", "it", "nl", "pl"] # ---------------------------------------------------------------- engine setup repo_dir = snapshot_download(MODEL_ID, allow_patterns=["oasis.py", "oasis.cpp", "oasis.oasis"]) build_dir = "/tmp/oasis_build" os.makedirs(build_dir, exist_ok=True) sys.path.insert(0, repo_dir) import oasis as oasis_mod # noqa: E402 (authors' binding) lib_path = os.path.join(build_dir, oasis_mod.LIBRARY) if not os.path.isfile(lib_path): t0 = time.time() oasis_mod.build(out=lib_path) print(f"[oasis] engine compiled in {time.time() - t0:.1f}s -> {lib_path}", flush=True) MODEL = oasis_mod.Oasis(os.path.join(repo_dir, "oasis.oasis"), library=lib_path) LOCK = threading.Lock() # one engine handle with a shared output buffer print(f"[oasis] engine loaded, OMP_NUM_THREADS={os.environ['OMP_NUM_THREADS']} (os.cpu_count={os.cpu_count()})", flush=True) # ---------------------------------------------------------------- drawing PALETTE = [ (230, 25, 75), (60, 180, 75), (0, 130, 200), (245, 130, 48), (145, 30, 180), (70, 240, 240), (240, 50, 230), (210, 245, 60), (0, 128, 128), (170, 110, 40), ] def _scale(pts, w, h): return [(pts[i] * w / 1000.0, pts[i + 1] * h / 1000.0) for i in range(0, len(pts) - 1, 2)] def _font(size): try: return ImageFont.truetype("DejaVuSans-Bold.ttf", size) except Exception: return ImageFont.load_default() def draw_record(image: Image.Image, record: dict) -> Image.Image: img = image.convert("RGB") w, h = img.size overlay = Image.new("RGBA", img.size, (0, 0, 0, 0)) od = ImageDraw.Draw(overlay) lw = max(2, round(min(w, h) / 300)) fs = max(12, round(min(w, h) / 45)) font = _font(fs) objects = record.get("objects") or [] for i, obj in enumerate(objects): c = PALETTE[i % len(PALETTE)] for poly in obj.get("mask") or []: if isinstance(poly, list) and len(poly) >= 6 and all(isinstance(v, (int, float)) for v in poly): od.polygon(_scale(poly, w, h), fill=c + (80,), outline=c + (220,)) img = Image.alpha_composite(img.convert("RGBA"), overlay).convert("RGB") d = ImageDraw.Draw(img) def label(x0, y0, text, c): tb = d.textbbox((x0, y0), text, font=font) y = y0 - (tb[3] - tb[1]) - 6 if y0 - (tb[3] - tb[1]) - 6 >= 0 else y0 tb = d.textbbox((x0 + 3, y + 2), text, font=font) d.rectangle([tb[0] - 3, tb[1] - 2, tb[2] + 3, tb[3] + 2], fill=c) d.text((x0 + 3, y + 2), text, fill=(255, 255, 255), font=font) for i, obj in enumerate(objects): b = obj.get("bbox") if not b or len(b) != 4: continue c = PALETTE[i % len(PALETTE)] x0, y0, x1, y1 = b[0] * w / 1000, b[1] * h / 1000, b[2] * w / 1000, b[3] * h / 1000 d.rectangle([x0, y0, x1, y1], outline=c, width=lw) score = obj.get("score") label(x0, y0, f"{obj.get('id', i + 1)} {obj.get('name', '?')}" + (f" {score:.2f}" if isinstance(score, (int, float)) else ""), c) for t in record.get("text") or []: b = t.get("bbox") if not b or len(b) != 4: continue c = (255, 200, 0) x0, y0, x1, y1 = b[0] * w / 1000, b[1] * h / 1000, b[2] * w / 1000, b[3] * h / 1000 d.rectangle([x0, y0, x1, y1], outline=c, width=lw) label(x1 + 4, y0 + fs + 6, f"“{t.get('string', '')}”", (40, 40, 40)) # form answers that are boxes form = record.get("form") or {} for i, (name, field) in enumerate(form.items()): v = field.get("value") if isinstance(field, dict) else None if isinstance(v, list) and len(v) == 4 and all(isinstance(x, (int, float)) for x in v): c = PALETTE[(i + 3) % len(PALETTE)] x0, y0, x1, y1 = v[0] * w / 1000, v[1] * h / 1000, v[2] * w / 1000, v[3] * h / 1000 d.rectangle([x0, y0, x1, y1], outline=c, width=lw + 1) label(x0, y0, name, c) return img def _load(image) -> Image.Image: if image is None: raise gr.Error("Please upload an image.") if isinstance(image, str): image = Image.open(image) from PIL import ImageOps return ImageOps.exif_transpose(image).convert("RGB") def _run(img: Image.Image, options: dict) -> dict: with LOCK: try: return MODEL.describe(img, **options) except RuntimeError as e: raise gr.Error(f"Oasis engine error: {e}") # ---------------------------------------------------------------- handlers def describe( image, language: str = "en", keywords: str = "", conf: float = 0.3, mask_format: str = "polygons", reasoning: bool = True, max_objects: int = 20, ): """Describe a photo as a structured JSON scene record. Args: image: the input photo (filepath). language: description language, one of en, de, fr, es, it, nl, pl. keywords: optional comma-separated words to steer the description toward. conf: object confidence threshold (0-1). mask_format: "polygons", "largest", "merged" or "none". reasoning: include the model's short reasoning trace. max_objects: maximum number of objects to report. Returns: An annotated image (boxes, masks, read text) and the JSON scene record. Boxes and outlines use a 0-1000 scale on each image axis. """ img = _load(image) options = { "language": language, "conf": float(conf), "max_objects": int(max_objects), "reasoning": bool(reasoning), } if mask_format == "none": options["masks"] = False else: options["mask_format"] = mask_format kw = [k.strip() for k in (keywords or "").split(",") if k.strip()] if kw: options["keywords"] = kw record = _run(img, options) return draw_record(img, record), record DEFAULT_FORM = json.dumps( { "vehicle": {"enum": ["motorcycle", "bicycle", "car", "truck"]}, "color": {"enum": ["red", "blue", "black", "white"]}, "parked_cars": {"type": "integer", "minimum": 0, "maximum": 10}, "where": {"x-source": "bbox", "x-of": "vehicle"}, }, indent=2, ) def fill_form(image, form_json: str = DEFAULT_FORM): """Fill an ordered form about a photo. Fields: {"name": {"enum": [...]}} for a choice, {"type": "integer", "minimum": 0, "maximum": 9} for a count, {"type": "string"} for free text, and {"x-source": "bbox", "x-of": ""} for a box. Args: image: the input photo (filepath). form_json: the form schema as JSON text. Returns: An annotated image (box answers drawn) and the filled form with per-answer probabilities. """ img = _load(image) try: form = json.loads(form_json) except json.JSONDecodeError as e: raise gr.Error(f"Form is not valid JSON: {e}") if not isinstance(form, dict) or not form: raise gr.Error("Form must be a non-empty JSON object of fields.") record = _run(img, {"form": form}) return draw_record(img, record), record # ---------------------------------------------------------------- UI CSS = """ #col-container { max-width: 1200px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(title="Oasis scene parser") as demo: with gr.Column(elem_id="col-container"): gr.Markdown( "# 🏝️ Oasis — photo → structured scene JSON\n" "A 19 MB, 2-bit vision model (Cactus Compute's Mirage with a new CPU engine and a retrained text reader) " "that turns a photo into a JSON record: objects with boxes and masks, text it can read, relations and a " "short reasoning trace — or fills in a form you define. Runs on CPU. " "[Model](https://huggingface.co/phanerozoic/oasis) · [Base model: Mirage](https://huggingface.co/Cactus-Compute/mirage)" ) with gr.Tabs(): with gr.Tab("Describe a photo"): with gr.Row(): with gr.Column(): d_image = gr.Image(type="filepath", label="Photo", height=420) d_btn = gr.Button("Describe", variant="primary") with gr.Accordion("Options", open=False): d_lang = gr.Dropdown(LANGUAGES, value="en", label="Description language") d_kw = gr.Textbox(label="Keywords (comma-separated, optional)", placeholder="e.g. sign, car") d_conf = gr.Slider(0.05, 0.95, value=0.3, step=0.05, label="Object confidence threshold") d_mask = gr.Radio(["polygons", "largest", "merged", "none"], value="polygons", label="Mask format") d_reason = gr.Checkbox(value=True, label="Include reasoning trace") d_maxobj = gr.Slider(1, 64, value=20, step=1, label="Max objects") with gr.Column(): d_out_img = gr.Image(label="Annotated", height=420) d_out_json = gr.JSON(label="Scene record") gr.Examples( examples=[ ["examples/sign_beach.jpg"], ["examples/motorcycle_street.jpg"], ["examples/girl_with_dog.jpg"], ["examples/cafe_interior.jpg"], ["examples/living_room_brick.jpg"], ], inputs=[d_image], outputs=[d_out_img, d_out_json], fn=describe, cache_examples=True, cache_mode="lazy", ) d_btn.click( describe, inputs=[d_image, d_lang, d_kw, d_conf, d_mask, d_reason, d_maxobj], outputs=[d_out_img, d_out_json], api_name="describe", ) with gr.Tab("Fill a form"): with gr.Row(): with gr.Column(): f_image = gr.Image(type="filepath", label="Photo", height=420) f_form = gr.Code(value=DEFAULT_FORM, language="json", label="Form (ordered fields)") f_btn = gr.Button("Fill form", variant="primary") gr.Markdown( "Field types: `{\"enum\": [...]}` choice · `{\"type\": \"integer\", \"minimum\": 0, \"maximum\": 9}` count · " "`{\"type\": \"string\"}` text · `{\"x-source\": \"bbox\", \"x-of\": \"\"}` box." ) with gr.Column(): f_out_img = gr.Image(label="Annotated", height=420) f_out_json = gr.JSON(label="Filled form") gr.Examples( examples=[ ["examples/motorcycle_street.jpg", DEFAULT_FORM], [ "examples/sign_beach.jpg", json.dumps( { "sign": {"enum": ["stop sign", "letter board", "billboard"]}, "place": {"enum": ["beach", "city", "forest", "kitchen"]}, "cars": {"type": "integer", "minimum": 0, "maximum": 10}, "where": {"x-source": "bbox", "x-of": "sign"}, }, indent=2, ), ], [ "examples/girl_with_dog.jpg", json.dumps( { "animal": {"enum": ["dog", "cat", "horse", "bird"]}, "animals": {"type": "integer", "minimum": 0, "maximum": 10}, "people": {"type": "integer", "minimum": 0, "maximum": 10}, "where": {"x-source": "bbox", "x-of": "animal"}, }, indent=2, ), ], ], inputs=[f_image, f_form], outputs=[f_out_img, f_out_json], fn=fill_form, cache_examples=True, cache_mode="lazy", ) f_btn.click( fill_form, inputs=[f_image, f_form], outputs=[f_out_img, f_out_json], api_name="fill_form", ) if __name__ == "__main__": demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)