curbcheck: read-then-resolve VLM parking checker

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/dpw_0accb82f29.jpg filter=lfs diff=lfs merge=lfs -text
+examples/dpw_0b0dc146cc.jpg filter=lfs diff=lfs merge=lfs -text
+examples/dpw_0b6a0cec08.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,58 @@
 ---
-title: Curbcheck
-emoji: ⚡
-colorFrom: pink
-colorTo: yellow
+title: curbcheck
+emoji: 🅿️
+colorFrom: red
+colorTo: gray
 sdk: gradio
-sdk_version: 6.18.0
-python_version: '3.12'
 app_file: app.py
-pinned: false
+pinned: true
+license: mit
+tags:
+  - build-small-hackathon
+  - backyard-ai
+  - vision-language-model
+  - qlora
+  - qwen2.5-vl
+short_description: Can a small VLM tell you if you can legally park in SF?
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🅿️ curbcheck
+
+**Can a small vision-language model tell you if you can legally park in San Francisco?**
+
+Upload a photo of a parking-sign pole, pick a day and time, and a QLoRA-tuned
+**Qwen2.5-VL-3B** reads each sign into structured rules. A tiny deterministic resolver
+then applies them to that moment and returns the verdict, showing you *both* what the
+model read and why. Read-then-resolve: the VLM only perceives; the logic is exact.
+
+I came to SF for a week in April 2026 and left with two parking tickets, both because I
+couldn't parse a pole holding four stacked signs. curbcheck is the model that gets it.
+
+## Track & badges
+
+- **Track:** Backyard AI (a real, local, personal problem solved with a small model)
+- A 3B model, well under the 32B cap, runs on ZeroGPU.
+
+## How it works
+
+```
+photo ─▶ VLM (reads each sign to JSON) ─▶ deterministic resolver ─▶ verdict + reason
+```
+
+- The model only **reads** the pole into structured restrictions (kind, days, hours, limits, permits).
+- A deterministic resolver (`rules.py`, no model in the loop) applies them to the current time.
+- Both halves are shown, so misreads are visible, not buried in a confident sentence.
+
+## The result
+
+A stock Qwen2.5-VL-3B scores 0.16 on "can I park here right now", below random. One QLoRA
+run takes it to **0.82 reasoning** and **0.98 read accuracy** on the synthetic benchmark; on
+real SF photos the read-then-resolve pipeline reasons at **0.89**.
+
+Full benchmark, training, and honest results: **https://github.com/shubhamgoel27/curbcheck**
+
+## Links
+
+- Code + benchmark: https://github.com/shubhamgoel27/curbcheck
+- Demo video: _(add link)_
+- Social post: _(add link)_

app.py

@@ -0,0 +1,204 @@
+"""curbcheck demo: can a small VLM tell you if you can legally park in SF?
+
+Upload a photo of a parking-sign pole, pick a day/time, and the model reads each
+sign into structured rules, then a deterministic resolver applies them to that
+moment and returns a verdict, with its reasoning shown. Read-then-resolve: the
+VLM only perceives, the logic is exact.
+
+Qwen2.5-VL-3B + a QLoRA adapter (curbcheck v4), on ZeroGPU.
+"""
+
+import json
+import os
+import re
+from datetime import datetime, time as dtime
+
+import gradio as gr
+import spaces
+import torch
+from PIL import Image
+from peft import PeftModel
+from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration
+from qwen_vl_utils import process_vision_info
+
+from rules import Kind, Day, Restriction, Window, SignStack, can_park, Verdict
+
+BASE_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
+ADAPTER_REPO = os.environ.get("ADAPTER_REPO", "shubhamgoel27/curbcheck-qwen25vl3b-v4-lora")
+
+READ_PROMPT = """Look at the parking sign stack in this image. Extract EVERY sign as a JSON array.
+Each element: {"kind": one of [no_parking, no_stopping, tow_away, time_limit, permit_limit, street_cleaning, loading_only, angle_parking] (use permit_limit when the sign has a permit exemption like EXCEPT AREA X PERMIT, time_limit otherwise; use angle_parking for orientation signs like "PARK AT 90 DEGREES" which do not restrict parking),
+"days": list like ["MON","TUE"...] (the days the restriction applies, null for angle_parking),
+"start": "HH:MM" 24h, "end": "HH:MM" 24h,
+"limit_minutes": int or null, "permit_area": letter or null, "tow": true/false,
+"weeks": list of which weeks of the month it applies like [2,4] for "2nd & 4th MONDAY" (works on ANY sign type, not just cleaning), or null for every week}.
+Respond with ONLY the JSON array, nothing else."""
+
+# ---- load model once (CPU at startup; moved to GPU inside the @spaces.GPU fn) ----
+print("loading base + adapter...")
+processor = AutoProcessor.from_pretrained(BASE_ID)
+model = Qwen2_5_VLForConditionalGeneration.from_pretrained(BASE_ID, torch_dtype=torch.bfloat16)
+model = PeftModel.from_pretrained(model, ADAPTER_REPO)
+model.eval()
+print("model ready")
+
+dec = json.JSONDecoder()
+
+
+def extract(t):
+    t = re.sub(r"```(?:json)?", "", t).strip("` \n")
+    for i, ch in enumerate(t):
+        if ch in "[{":
+            try:
+                return dec.raw_decode(t[i:])[0]
+            except json.JSONDecodeError:
+                continue
+    return None
+
+
+def build_stack(read_json):
+    out = []
+    for r in read_json if isinstance(read_json, list) else []:
+        if not isinstance(r, dict):
+            continue
+        try:
+            kind = Kind(str(r["kind"]).lower().replace("-", "_"))
+            if kind is Kind.ANGLE_PARKING:
+                out.append(Restriction(kind, Window(frozenset(), dtime(0), dtime(0))))
+                continue
+            days = frozenset(Day[str(d)[:3].upper()] for d in (r.get("days") or []))
+            sh, sm = map(int, str(r["start"]).split(":"))
+            eh, em = map(int, str(r["end"]).split(":"))
+            wk = frozenset(int(x) for x in (r.get("weeks") or []))
+            out.append(Restriction(kind, Window(days, dtime(sh, sm), dtime(eh, em), weeks=wk),
+                                   limit_minutes=r.get("limit_minutes"),
+                                   permit_area=r.get("permit_area"), tow=bool(r.get("tow"))))
+        except Exception:
+            continue
+    return SignStack(out)
+
+
+VERDICT_UI = {
+    Verdict.OK: ("✅ You can park", "#0a7d3c"),
+    Verdict.LIMITED: ("⏱️ Limited parking", "#b8860b"),
+    Verdict.NO: ("🚫 No parking", "#c1452a"),
+    Verdict.TOW_RISK: ("🚨 Tow risk", "#8b0000"),
+    Verdict.ABSTAIN: ("🤔 Can't tell from the sign", "#555"),
+}
+
+
+@spaces.GPU(duration=90)
+def read_signs(image):
+    model.to("cuda")
+    msgs = [{"role": "user", "content": [
+        {"type": "image", "image": image}, {"type": "text", "text": READ_PROMPT}]}]
+    text = processor.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
+    imgs, vids = process_vision_info(msgs)
+    inp = processor(text=[text], images=imgs, return_tensors="pt").to("cuda")
+    with torch.no_grad():
+        out = model.generate(**inp, max_new_tokens=400, do_sample=False)
+    trim = out[0][inp.input_ids.shape[1]:]
+    return processor.decode(trim, skip_special_tokens=True)
+
+
+DOW = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
+KIND_LABEL = {
+    Kind.NO_PARKING: "No parking", Kind.NO_STOPPING: "No stopping",
+    Kind.TOW_AWAY: "Tow away", Kind.TIME_LIMIT: "Time limit",
+    Kind.STREET_CLEANING: "Street cleaning", Kind.PERMIT_EXEMPT_LIMIT: "Permit / time limit",
+    Kind.LOADING_ONLY: "Loading only", Kind.ANGLE_PARKING: "Angle parking (info)",
+}
+
+
+def fmt_restriction(r):
+    if r.kind is Kind.ANGLE_PARKING:
+        return f"- **{KIND_LABEL[r.kind]}** (does not restrict parking)"
+    days = ", ".join(d.name.title() for d in sorted(r.window.days, key=lambda d: d.value)) or "every day"
+    span = f"{r.window.start.strftime('%-I:%M%p').lower()}–{r.window.end.strftime('%-I:%M%p').lower()}"
+    bits = [f"**{KIND_LABEL.get(r.kind, r.kind.value)}**", span, days]
+    if r.window.weeks:
+        bits.append("weeks " + "/".join(str(w) for w in sorted(r.window.weeks)) + " of month")
+    if r.limit_minutes:
+        bits.append(f"{r.limit_minutes}min limit")
+    if r.permit_area:
+        bits.append(f"except Area {r.permit_area} permit")
+    if r.tow:
+        bits.append("TOW")
+    return "- " + " · ".join(bits)
+
+
+def predict(image, day, hour, minute, ampm, permit):
+    if image is None:
+        return "### Upload a photo of a parking sign first.", "", ""
+    raw = read_signs(image)
+    parsed = extract(raw)
+    stack = build_stack(parsed)
+
+    # build the "when"
+    h = int(hour) % 12 + (12 if ampm == "PM" else 0)
+    dow_idx = DOW.index(day)
+    # next date matching that weekday (anchored to 2026-06-15, a Monday)
+    base = datetime(2026, 6, 15, h, int(minute))
+    when = base.replace(day=15 + ((dow_idx - 0) % 7))
+
+    permit_set = frozenset(p.strip().upper() for p in permit.split(",") if p.strip())
+    ans = can_park(stack, when, permit_areas=permit_set)
+
+    label, color = VERDICT_UI.get(ans.verdict, ("?", "#555"))
+    detail = ""
+    if ans.verdict is Verdict.LIMITED and ans.limit_minutes:
+        detail = f" — up to {ans.limit_minutes} minutes"
+    verdict_md = (
+        f"<div style='font-size:1.6em;font-weight:700;color:{color}'>{label}{detail}</div>"
+        f"<div style='color:#666;margin-top:6px'>on {day} at {int(hour)}:{int(minute):02d} {ampm}"
+        + (f", with permit {','.join(permit_set)}" if permit_set else ", no permit") + "</div>"
+        f"<div style='margin-top:8px'>{ans.reason}</div>"
+    )
+
+    if stack.restrictions:
+        signs_md = "### What the model read on the pole\n" + "\n".join(
+            fmt_restriction(r) for r in stack.restrictions)
+    else:
+        signs_md = "### What the model read on the pole\n_No structured signs parsed._"
+
+    return verdict_md, signs_md, raw.strip()
+
+
+THEME = gr.themes.Soft(primary_hue="red", neutral_hue="stone")
+
+with gr.Blocks(theme=THEME, title="curbcheck") as demo:
+    gr.Markdown(
+        "# 🅿️ curbcheck\n"
+        "**Can a small VLM tell you if you can legally park in San Francisco?** "
+        "Upload a photo of a sign pole, pick a day and time, and a QLoRA-tuned "
+        "Qwen2.5-VL-3B reads each sign into structured rules. A deterministic resolver "
+        "then decides the verdict, so you see *both* what it read and why. "
+        "[Project + benchmark on GitHub](https://github.com/shubhamgoel27/curbcheck)."
+    )
+    with gr.Row():
+        with gr.Column(scale=1):
+            img = gr.Image(type="pil", label="Parking sign photo", height=360)
+            with gr.Row():
+                day = gr.Dropdown(DOW, value="Tuesday", label="Day")
+                hour = gr.Dropdown([str(i) for i in range(1, 13)], value="5", label="Hour")
+                minute = gr.Dropdown(["00", "15", "30", "45"], value="30", label="Min")
+                ampm = gr.Dropdown(["AM", "PM"], value="PM", label="")
+            permit = gr.Textbox(label="Your permit area(s), if any", placeholder="e.g. S")
+            btn = gr.Button("Can I park here?", variant="primary")
+        with gr.Column(scale=1):
+            verdict_out = gr.Markdown()
+            signs_out = gr.Markdown()
+            with gr.Accordion("Raw model output (JSON)", open=False):
+                raw_out = gr.Code(language="json")
+
+    btn.click(predict, [img, day, hour, minute, ampm, permit],
+              [verdict_out, signs_out, raw_out])
+
+    import glob
+    ex = sorted(glob.glob("examples/*.jpg"))[:4]
+    if ex:
+        gr.Examples([[e, "Tuesday", "5", "30", "PM", ""] for e in ex],
+                    [img, day, hour, minute, ampm, permit], label="Try a real SF photo")
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,7 @@
+spaces
+torch
+transformers>=4.49.0
+accelerate>=1.2.0
+peft>=0.14.0
+qwen-vl-utils
+pillow

rules.py

@@ -0,0 +1,122 @@
+"""The rule schema: what a San Francisco parking sign stack can say, as data.
+
+Design notes:
+- A pole holds a stack of Restriction objects (top to bottom matters for rendering,
+  not for semantics).
+- Answering "can I park here at time T?" applies every restriction independently;
+  the most severe applicable verdict wins (TOW > NO_PARK > NO_STOP > TIME_LIMIT > FREE).
+- v1 scope: sign-stack-only. Curb paint and meters are out of frame; queries that
+  depend on them must be answered ABSTAIN by a correct model.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, time
+from enum import Enum
+
+
+class Day(Enum):
+    MON, TUE, WED, THU, FRI, SAT, SUN = range(7)
+
+
+WEEKDAYS = frozenset({Day.MON, Day.TUE, Day.WED, Day.THU, Day.FRI})
+EVERY_DAY = frozenset(Day)
+
+
+class Kind(Enum):
+    NO_STOPPING = "no_stopping"            # CA R26(S): no stopping at any time in window
+    NO_PARKING = "no_parking"              # CA R26: no parking in window
+    TOW_AWAY = "tow_away"                  # modifier or standalone: violation = tow
+    TIME_LIMIT = "time_limit"              # CA R30: e.g. 2-hour parking 9am-6pm
+    STREET_CLEANING = "street_cleaning"    # CA R32: no parking, specific weekday window
+    PERMIT_EXEMPT_LIMIT = "permit_limit"   # RPP: time limit EXCEPT vehicles with area permit
+    LOADING_ONLY = "loading_only"          # passenger/commercial loading zone window
+    ANGLE_PARKING = "angle_parking"        # informational: "PARK AT 90 DEGREES" etc. No verdict effect.
+
+
+@dataclass(frozen=True)
+class Window:
+    """A recurring weekly time window, e.g. Tue 8:00-10:00."""
+    days: frozenset[Day]
+    start: time
+    end: time
+    weeks: frozenset[int] = frozenset()  # which weeks of the month (1-5); empty = every week
+
+    def contains(self, dt: datetime) -> bool:
+        if Day(dt.weekday()) not in self.days or not (self.start <= dt.time() < self.end):
+            return False
+        if self.weeks:  # "2nd & 4th Monday" style: nth occurrence of the weekday in the month
+            week_of_month = (dt.day - 1) // 7 + 1
+            if week_of_month not in self.weeks:
+                return False
+        return True
+
+
+@dataclass(frozen=True)
+class Restriction:
+    kind: Kind
+    window: Window
+    limit_minutes: int | None = None    # TIME_LIMIT / PERMIT_EXEMPT_LIMIT
+    permit_area: str | None = None      # RPP area letter, e.g. "S"
+    tow: bool = False                   # tow-away enforcement on violation
+
+
+@dataclass
+class SignStack:
+    """Everything on one pole."""
+    restrictions: list[Restriction] = field(default_factory=list)
+    pole_id: str | None = None
+
+
+class Verdict(Enum):
+    TOW_RISK = "tow_risk"           # parking now risks a tow
+    NO = "no"                       # no parking (citation risk)
+    LIMITED = "limited"             # ok up to N minutes
+    OK = "ok"                       # no restriction applies right now
+    ABSTAIN = "abstain"             # not decidable from the sign stack alone
+
+
+@dataclass
+class Answer:
+    verdict: Verdict
+    limit_minutes: int | None = None   # for LIMITED
+    until: datetime | None = None      # next moment the verdict changes (v2)
+    reason: str = ""
+
+
+SEVERITY = [Verdict.TOW_RISK, Verdict.NO, Verdict.LIMITED, Verdict.OK]
+
+
+def can_park(stack: SignStack, when: datetime, permit_areas: frozenset[str] = frozenset()) -> Answer:
+    """The resolver: ground truth for every generated question."""
+    verdicts: list[Answer] = []
+    for r in stack.restrictions:
+        if r.kind is Kind.ANGLE_PARKING:
+            continue  # informational ("park at 90 degrees"), never affects the verdict
+        if not r.window.contains(when):
+            continue
+        if r.kind in (Kind.NO_STOPPING, Kind.NO_PARKING, Kind.STREET_CLEANING, Kind.LOADING_ONLY):
+            v = Verdict.TOW_RISK if (r.tow or r.kind is Kind.NO_STOPPING) else Verdict.NO
+            verdicts.append(Answer(v, reason=f"{r.kind.value} in effect"))
+        elif r.kind is Kind.TIME_LIMIT:
+            verdicts.append(Answer(Verdict.LIMITED, limit_minutes=r.limit_minutes,
+                                   reason=f"{r.limit_minutes}min limit"))
+        elif r.kind is Kind.PERMIT_EXEMPT_LIMIT:
+            # permit_area must be a hashable scalar; malformed model output (e.g. a list)
+            # is treated as "no matching permit"
+            if isinstance(r.permit_area, str) and r.permit_area in permit_areas:
+                verdicts.append(Answer(Verdict.OK, reason=f"permit {r.permit_area} exempts"))
+            else:
+                verdicts.append(Answer(Verdict.LIMITED, limit_minutes=r.limit_minutes,
+                                       reason=f"{r.limit_minutes}min limit without permit {r.permit_area}"))
+    if not verdicts:
+        return Answer(Verdict.OK, reason="no restriction in effect")
+    verdicts.sort(key=lambda a: SEVERITY.index(a.verdict))
+    best = verdicts[0]
+    if best.verdict is Verdict.LIMITED:
+        # multiple limits: strictest applies. Tolerate None (malformed model reads).
+        limits = [a.limit_minutes for a in verdicts
+                  if a.verdict is Verdict.LIMITED and a.limit_minutes is not None]
+        best.limit_minutes = min(limits) if limits else None
+    return best