Add Self-Contrastive Grounding inference and ablations

Adds the six-prefill cache-reuse implementation, evaluator flag, full three-benchmark metrics, ablations, and reproducibility manifest. Model weights are unchanged.

README.md

@@ -36,6 +36,55 @@ element-grounding subsets. OSWorld-G uses its 510 target-bearing examples;
 refusal-only rows are excluded. These public benchmarks were used during model
 selection, so results are test-tuned rather than held-out validation estimates.
 
+## Self-Contrastive Grounding
+
+The release also includes Self-Contrastive Grounding, a training-free extension
+of the paper's contrast-mining principle. Training mines observed hard
+distractors from disagreement between models. At inference, Self-Contrast mines
+latent distractors from disagreement between deterministic views of the same
+model, then asks every other view to verify each visible coordinate. A proposal
+is never scored by the view that generated it, which prevents self-confirmation.
+
+| Inference | ScreenSpot-Pro | UI-Vision | OSWorld-G |
+|---|---:|---:|---:|
+| Direct | 65.09 | 37.12 | 69.41 |
+| Self-Contrast | **71.16** | **44.09** | **72.75** |
+
+The method uses one full-screen view, one 40% incumbent-centered revisit, and
+four fixed overlapping 60% views. All crops are enlarged by 2x. Within each
+view, coordinate-string mean token log-likelihoods are standardized; evidence
+is averaged across non-source views and combined at equal weight with proximity
+to the incumbent revisit. This one configuration is shared by all three
+benchmarks: there is no benchmark-specific gate, router, prompt, or threshold.
+
+The implementation retains each view's visual prefix after greedy candidate
+generation and reuses its KV cache for batched coordinate scoring. It therefore
+uses six visual prefills, rather than the twelve prefills of a naive
+generate-then-rescore implementation, and requires no weight update.
+
+```bash
+python evaluate.py \
+  --model ruotian/SelectGround-8B \
+  --benchmark screenspot_pro \
+  --data data/screenspot-pro \
+  --self-contrast \
+  --output outputs/screenspot-pro-self-contrast.jsonl
+```
+
+Full 8B ablations, using the same benchmark protocols, are:
+
+| Variant | ScreenSpot-Pro | UI-Vision | OSWorld-G |
+|---|---:|---:|---:|
+| Full | 71.16 | 44.09 | 72.75 |
+| no latent distractors | 71.22 | 43.44 | 69.61 |
+| one latent distractor | 70.97 | 43.44 | 70.59 |
+| no recurrent anchor | 68.82 | 43.23 | 72.75 |
+| no cross-view evidence | 71.16 | 43.46 | 69.41 |
+| no anchor proximity | 70.15 | 44.10 | 72.94 |
+
+`self_contrast_manifest.json` records the exact protocol, full counts, split
+metrics, artifact checksums, and cache-reuse equivalence test.
+
 ## Direct inference
 
 The repository includes the exact loader and evaluator. `visual_merger.pt` must

evaluate.py

@@ -7,6 +7,7 @@ from pathlib import Path
 from PIL import Image
 
 from selectground import SelectGround
+from self_contrast import SelfContrastGrounder
 
 
 def load_cases(name: str, root: Path):
@@ -117,6 +118,19 @@ parser.add_argument("--benchmark", choices=("screenspot_pro", "ui_vision", "oswo
 parser.add_argument("--data", type=Path, required=True)
 parser.add_argument("--output", type=Path, required=True)
 parser.add_argument("--lcr", action="store_true")
+parser.add_argument("--self-contrast", action="store_true")
+parser.add_argument(
+    "--self-contrast-variant",
+    choices=(
+        "full",
+        "no_latent_distractors",
+        "one_latent_distractor",
+        "no_recurrent_anchor",
+        "no_cross_view_evidence",
+        "no_anchor_proximity",
+    ),
+    default="full",
+)
 parser.add_argument(
     "--lcr-variant",
     choices=("full", "no_competitor", "one_competitor", "no_incumbent"),
@@ -126,6 +140,8 @@ parser.add_argument("--limit", type=int)
 parser.add_argument("--num-shards", type=int, default=1)
 parser.add_argument("--shard", type=int, default=0)
 args = parser.parse_args()
+if args.lcr and args.self_contrast:
+    parser.error("--lcr and --self-contrast are mutually exclusive")
 
 cases = (
     case for index, case in enumerate(load_cases(args.benchmark, args.data))
@@ -137,19 +153,26 @@ existing = []
 if args.output.exists():
     existing = [json.loads(line) for line in args.output.read_text().splitlines() if line.strip()]
 done = {row["id"] for row in existing}
-grounder = SelectGround(args.model)
+grounder = SelfContrastGrounder(args.model) if args.self_contrast else SelectGround(args.model)
 args.output.parent.mkdir(parents=True, exist_ok=True)
 with args.output.open("a") as output:
     for number, case in enumerate(cases, 1):
         if case["id"] in done:
             continue
-        prediction = grounder.predict(
-            case["image"],
-            case["instruction"],
-            lcr=args.lcr,
-            benchmark=args.benchmark,
-            lcr_variant=args.lcr_variant,
-        )
+        if args.self_contrast:
+            prediction = grounder.predict(
+                case["image"],
+                case["instruction"],
+                variant=args.self_contrast_variant,
+            )
+        else:
+            prediction = grounder.predict(
+                case["image"],
+                case["instruction"],
+                lcr=args.lcr,
+                benchmark=args.benchmark,
+                lcr_variant=args.lcr_variant,
+            )
         row = {
             "id": case["id"],
             "instruction": case["instruction"],

self_contrast.py

@@ -0,0 +1,328 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+import math
+from pathlib import Path
+from typing import Any
+
+from PIL import Image
+import torch
+from transformers.cache_utils import DynamicCache
+
+from selectground import SelectGround, _map_crop, _prediction
+
+
+GRID_CENTERS = ((0.3, 0.3), (0.7, 0.3), (0.3, 0.7), (0.7, 0.7))
+
+
+@dataclass
+class _Prefix:
+    cache: DynamicCache
+    logits: torch.Tensor
+    position_ids: torch.Tensor
+    attention_mask: torch.Tensor
+    length: int
+
+
+class SelfContrastGrounder:
+    """Training-free self-contrastive grounding with six visual prefills."""
+
+    def __init__(self, checkpoint: str = "ruotian/SelectGround-8B") -> None:
+        self.grounder = SelectGround(checkpoint)
+
+    def predict(
+        self,
+        image: str | Path | Image.Image,
+        instruction: str,
+        *,
+        variant: str = "full",
+    ) -> dict[str, Any]:
+        variants = {
+            "full": (GRID_CENTERS, True, True, 1.0),
+            "no_latent_distractors": ((), True, True, 1.0),
+            "one_latent_distractor": (GRID_CENTERS[:1], True, True, 1.0),
+            "no_recurrent_anchor": (GRID_CENTERS, False, True, 1.0),
+            "no_cross_view_evidence": (GRID_CENTERS, True, False, 1.0),
+            "no_anchor_proximity": (GRID_CENTERS, True, True, 0.0),
+        }
+        if variant not in variants:
+            raise ValueError(f"unknown self-contrast variant: {variant}")
+        grid_centers, use_anchor, use_evidence, proximity_weight = variants[variant]
+        source = (
+            Image.open(image).convert("RGB")
+            if not isinstance(image, Image.Image)
+            else image.convert("RGB")
+        )
+        full_box = (0, 0, source.width, source.height)
+        views: dict[str, tuple[tuple[int, int, int, int], Image.Image]] = {
+            "full": (full_box, source)
+        }
+        candidates = []
+        prefixes = {}
+
+        p0, prefixes["full"] = self._observe(source, instruction)
+        candidates.append(self._candidate("p0", p0, full_box, source.size))
+        if use_anchor and p0["point"] is not None:
+            box = _crop_box(tuple(p0["point"]), source.size, 0.40)
+            views["q0"] = (box, _view(source, box))
+        for index, center in enumerate(grid_centers):
+            point = (center[0] * source.width, center[1] * source.height)
+            box = _crop_box(point, source.size, 0.60)
+            views[f"grid_{index}"] = (box, _view(source, box))
+
+        for name, (box, view) in tuple(views.items())[1:]:
+            prediction, prefixes[name] = self._observe(view, instruction)
+            candidates.append(self._candidate(name, prediction, box, source.size))
+
+        evidence = {}
+        for view_name, (box, _) in views.items():
+            visible = {
+                candidate["name"]: response
+                for candidate in candidates
+                if candidate["point"] is not None
+                and (response := _response(candidate["point"], box)) is not None
+            }
+            scores = self._score(prefixes.pop(view_name), list(visible.values()))
+            evidence[view_name] = {
+                name: {"response": response, **scores[response]}
+                for name, response in visible.items()
+            }
+        selected = _select(
+            candidates,
+            evidence,
+            source.size,
+            proximity_weight=proximity_weight,
+            use_evidence=use_evidence,
+        )
+        point = selected["point"]
+        normalized = (
+            [1000 * point[0] / source.width, 1000 * point[1] / source.height]
+            if point is not None
+            else None
+        )
+        return {
+            "method": "SelectGround+SelfContrast",
+            "variant": variant,
+            "point": point,
+            "normalized_point": normalized,
+            "raw_response": selected["raw_response"],
+            "selected_candidate": selected["name"],
+        }
+
+    def _candidate(
+        self,
+        name: str,
+        prediction: dict[str, Any],
+        box: tuple[int, int, int, int],
+        source_size: tuple[int, int],
+    ) -> dict[str, Any]:
+        mapped = (
+            prediction
+            if name == "p0" or prediction["point"] is None
+            else _map_crop(prediction, box, source_size, 2.0)
+        )
+        return {
+            "name": name,
+            "point": mapped["point"],
+            "source_view": "full" if name == "p0" else name,
+            "raw_response": prediction["raw_response"],
+        }
+
+    @torch.inference_mode()
+    def _observe(
+        self, image: Image.Image, instruction: str
+    ) -> tuple[dict[str, Any], _Prefix]:
+        inputs = self.grounder._inputs(image, instruction, False)
+        input_ids = inputs["input_ids"]
+        length = int(input_ids.shape[1])
+        position_ids, _ = self.grounder.core.get_rope_index(
+            input_ids,
+            inputs.get("image_grid_thw"),
+            inputs.get("video_grid_thw"),
+            attention_mask=inputs.get("attention_mask"),
+        )
+        cache = DynamicCache(config=self.grounder.core.language_model.config)
+        output = self.grounder.model(
+            **inputs,
+            past_key_values=cache,
+            position_ids=position_ids,
+            cache_position=torch.arange(length, device=self.grounder.device),
+            use_cache=True,
+            logits_to_keep=1,
+        )
+        logits = output.logits[:, -1, :].detach()
+        raw = self.grounder._decode(
+            logits, cache, position_ids[:, :, -1:] + 1, None
+        )
+        cache.crop(length)
+        if cache.get_seq_length() != length:
+            raise RuntimeError("could not restore the visual prefix after decoding")
+        return (
+            _prediction(raw, image.size, integer=False),
+            _Prefix(cache, logits, position_ids, inputs["attention_mask"], length),
+        )
+
+    @torch.inference_mode()
+    def _score(
+        self, prefix: _Prefix, responses: list[str]
+    ) -> dict[str, dict[str, float | int]]:
+        unique = list(dict.fromkeys(responses))
+        if not unique:
+            return {}
+        encoded = [_token_ids(self.grounder, response) for response in unique]
+        first = torch.log_softmax(prefix.logits.float(), -1)
+        logps = [[float(first[0, values[0]])] for values in encoded]
+        maximum = max(map(len, encoded))
+        if maximum > 1:
+            tokenizer = self.grounder.processor.tokenizer
+            pad = tokenizer.pad_token_id or tokenizer.eos_token_id
+            continuation = torch.full(
+                (len(encoded), maximum - 1),
+                int(pad),
+                dtype=torch.long,
+                device=self.grounder.device,
+            )
+            mask = torch.zeros_like(continuation, dtype=torch.bool)
+            for index, values in enumerate(encoded):
+                if len(values) > 1:
+                    continuation[index, : len(values) - 1] = torch.tensor(
+                        values[:-1], device=self.grounder.device
+                    )
+                    mask[index, : len(values) - 1] = True
+            prefix.cache.batch_repeat_interleave(len(encoded))
+            positions = prefix.position_ids.repeat_interleave(len(encoded), dim=-2)
+            offsets = torch.arange(maximum - 1, device=self.grounder.device).view(
+                *([1] * (positions.ndim - 1)), -1
+            )
+            output = self.grounder.model(
+                input_ids=continuation,
+                past_key_values=prefix.cache,
+                attention_mask=torch.cat(
+                    (prefix.attention_mask.repeat(len(encoded), 1), mask.long()), 1
+                ),
+                position_ids=positions[..., -1:] + 1 + offsets,
+                cache_position=torch.arange(
+                    prefix.length,
+                    prefix.length + maximum - 1,
+                    device=self.grounder.device,
+                ),
+                use_cache=True,
+            )
+            for index, values in enumerate(encoded):
+                if len(values) <= 1:
+                    continue
+                logits = output.logits[index, : len(values) - 1].float()
+                labels = torch.tensor(values[1:], device=self.grounder.device)
+                selected = torch.log_softmax(logits, -1).gather(1, labels[:, None])[:, 0]
+                logps[index].extend(float(value) for value in selected)
+        return {
+            response: {
+                "token_count": len(values),
+                "mean_logprob": sum(values_logps) / len(values),
+            }
+            for response, values, values_logps in zip(unique, encoded, logps, strict=True)
+        }
+
+
+def _crop_box(
+    point: tuple[float, float], size: tuple[int, int], fraction: float
+) -> tuple[int, int, int, int]:
+    width, height = size
+    crop_width = min(width, max(320, round(fraction * width)))
+    crop_height = min(height, max(320, round(fraction * height)))
+    left = round(min(max(0.0, point[0] - crop_width / 2), width - crop_width))
+    top = round(min(max(0.0, point[1] - crop_height / 2), height - crop_height))
+    return left, top, left + crop_width, top + crop_height
+
+
+def _view(source: Image.Image, box: tuple[int, int, int, int]) -> Image.Image:
+    crop = source.crop(box)
+    return crop.resize(
+        (2 * crop.width, 2 * crop.height), Image.Resampling.LANCZOS
+    )
+
+
+def _response(point: list[float], box: tuple[int, int, int, int]) -> str | None:
+    left, top, right, bottom = box
+    x, y = map(float, point)
+    if not left <= x < right or not top <= y < bottom:
+        return None
+    return (
+        f"[{round(1000 * (x - left) / (right - left))},"
+        f"{round(1000 * (y - top) / (bottom - top))}]"
+    )
+
+
+def _token_ids(grounder: SelectGround, response: str) -> list[int]:
+    values = grounder.processor.tokenizer(
+        response, add_special_tokens=False
+    )["input_ids"]
+    if values and isinstance(values[0], list):
+        values = values[0]
+    result = [int(value) for value in values]
+    if not result:
+        raise ValueError(f"empty tokenization for {response!r}")
+    return result
+
+
+def _zscore(values: list[float]) -> list[float]:
+    mean = sum(values) / len(values)
+    std = math.sqrt(sum((value - mean) ** 2 for value in values) / len(values))
+    return [(value - mean) / max(std, 1e-6) for value in values]
+
+
+def _select(
+    candidates: list[dict[str, Any]],
+    evidence: dict[str, dict[str, dict[str, Any]]],
+    size: tuple[int, int],
+    *,
+    proximity_weight: float,
+    use_evidence: bool,
+) -> dict[str, Any]:
+    eligible = [candidate for candidate in candidates if candidate["point"] is not None]
+    if not eligible:
+        return candidates[0]
+    by_name = {candidate["name"]: candidate for candidate in eligible}
+    accumulated = defaultdict(list)
+    for view_name, values in evidence.items():
+        unique = {}
+        for name, value in values.items():
+            if name in by_name:
+                unique.setdefault(value["response"], float(value["mean_logprob"]))
+        if not unique:
+            continue
+        normalized = dict(zip(unique, _zscore(list(unique.values())), strict=True))
+        for name, value in values.items():
+            if name in by_name and by_name[name]["source_view"] != view_name:
+                accumulated[name].append(normalized[value["response"]])
+    if use_evidence:
+        eligible = [candidate for candidate in eligible if accumulated[candidate["name"]]]
+        if not eligible:
+            return candidates[0]
+        likelihood = _zscore(
+            [
+                sum(accumulated[candidate["name"]])
+                / len(accumulated[candidate["name"]])
+                for candidate in eligible
+            ]
+        )
+    else:
+        likelihood = [0.0] * len(eligible)
+    by_name = {candidate["name"]: candidate for candidate in eligible}
+    anchor = by_name.get("q0", by_name.get("p0", eligible[0]))["point"]
+    width, height = size
+    proximity = _zscore(
+        [
+            -math.hypot(
+                (candidate["point"][0] - anchor[0]) / width,
+                (candidate["point"][1] - anchor[1]) / height,
+            )
+            for candidate in eligible
+        ]
+    )
+    scores = [
+        likelihood[index] + proximity_weight * proximity[index]
+        for index in range(len(eligible))
+    ]
+    return eligible[max(range(len(eligible)), key=lambda index: (scores[index], -index))]

self_contrast_manifest.json

@@ -0,0 +1,81 @@
+{
+  "schema_version": "selectground.self_contrast.v1",
+  "method": "Self-Contrastive Grounding",
+  "training_free": true,
+  "model": {
+    "repo": "ruotian/SelectGround-8B",
+    "weight_source_revision": "a3043b9d3bcd00779bbf2f293f18dd5651e433cc",
+    "plain_base_start": true,
+    "aggregate": false,
+    "training_manifest_sha256": "f0352e027ffc99b69fc93c3960163702742abe6f8aa2d5999284ad717f21bb87"
+  },
+  "protocol": {
+    "prompt": "direct SelectGround prompt",
+    "decoding": "greedy",
+    "max_new_tokens": 32,
+    "views": [
+      {"name": "full", "crop_fraction": 1.0, "scale": 1.0},
+      {"name": "q0", "center": "initial prediction", "crop_fraction": 0.4, "scale": 2.0},
+      {"name": "grid_0", "center": [0.3, 0.3], "crop_fraction": 0.6, "scale": 2.0},
+      {"name": "grid_1", "center": [0.7, 0.3], "crop_fraction": 0.6, "scale": 2.0},
+      {"name": "grid_2", "center": [0.3, 0.7], "crop_fraction": 0.6, "scale": 2.0},
+      {"name": "grid_3", "center": [0.7, 0.7], "crop_fraction": 0.6, "scale": 2.0}
+    ],
+    "coordinate_evidence": "mean token log probability",
+    "within_view_normalization": "population z-score over unique visible coordinate strings",
+    "cross_view_aggregation": "mean over visible non-source views, then population z-score over candidates",
+    "source_view_excluded": true,
+    "anchor": "q0, falling back to p0",
+    "proximity_weight": 1.0,
+    "p0_prior": 0.0,
+    "parameter_scope": "one shared configuration for all benchmarks",
+    "visual_prefills": 6,
+    "visual_prefix_cache_reused": true,
+    "additional_training": false,
+    "router_or_gate": false
+  },
+  "evaluation": {
+    "test_tuned": true,
+    "screenspot_pro": {"total": 1581, "correct": 1125, "accuracy_pct": 71.15749525616698},
+    "ui_vision": {
+      "total": 5479,
+      "correct_micro": 2398,
+      "accuracy_macro_pct": 44.09020207146093,
+      "splits": {
+        "basic": {"total": 1772, "correct": 888, "accuracy_pct": 50.112866817155755},
+        "functional": {"total": 1772, "correct": 867, "accuracy_pct": 48.92776523702032},
+        "spatial": {"total": 1935, "correct": 643, "accuracy_pct": 33.229974160206716}
+      }
+    },
+    "osworld_g": {"total": 510, "correct": 371, "accuracy_pct": 72.74509803921569}
+  },
+  "ablation_accuracy_pct": {
+    "full": {"screenspot_pro": 71.15749525616698, "ui_vision": 44.09020207146093, "osworld_g": 72.74509803921569},
+    "no_latent_distractors": {"screenspot_pro": 71.22074636306135, "ui_vision": 43.43973534140997, "osworld_g": 69.6078431372549},
+    "one_latent_distractor": {"screenspot_pro": 70.96774193548387, "ui_vision": 43.43677027859925, "osworld_g": 70.58823529411765},
+    "no_recurrent_anchor": {"screenspot_pro": 68.81720430107528, "ui_vision": 43.22846732500783, "osworld_g": 72.74509803921569},
+    "no_cross_view_evidence": {"screenspot_pro": 71.15749525616698, "ui_vision": 43.45696187026441, "osworld_g": 69.41176470588235},
+    "no_anchor_proximity": {"screenspot_pro": 70.14547754585705, "ui_vision": 44.09516004534115, "osworld_g": 72.94117647058823}
+  },
+  "artifacts": {
+    "metrics_sha256": "33549379cfb984f200b6b3d1e464a89ac872c3dc439b28bdc720a60b542a596c",
+    "predictions_sha256": "f5214508ff44c79b0e9aa5e343b18f7d05dc2929a0b4ff23dfbf51c1b3def304",
+    "raw_evidence_sha256": {
+      "screenspot_pro": "cef8b52eef5932a84f8e26d00958532dfc8bd12e21a3d45733bd2d850353629c",
+      "ui_vision": "a52470cc7ca2f79b2d2d6b1f6fecf9df45b0505248fd1776dc446dad7900ebca",
+      "osworld_g": "b1edf2c473795828fa8de67558dfe19dbc9ee1295d42310dbc5dbf74fb43b4fc"
+    },
+    "raw_evidence_runner_sha256": "21e6392a8b36bc8b78f1222fa6f4b89e9bea2a35ed75e15dc2a388b147e9f6fe",
+    "optimized_eval_runner_sha256": "8671248c9f015aa2046b5fe238259f61c0dba74d1a69c52b9c00b3d89bb31e29",
+    "scorer_sha256": "773aba1d2017da538e03c6436b3a2bb948f6714179019ab0a502c2769ac15000",
+    "release_runner_sha256": "56fab886aca1691c18de7b483ac9883e932dcb96b09cffe153c1fb2e25624ad5",
+    "release_evaluator_sha256": "a8a4993273a098a578895c53beda15210f74aa118edb95f09aea028fc92ea283"
+  },
+  "optimized_equivalence_check": {
+    "shared_cases": 125,
+    "candidate_generation_exact": 125,
+    "cross_view_evidence_exact": 125,
+    "hardware": "NVIDIA RTX A6000",
+    "note": "The six-prefill cache-reuse implementation was exactly equal to the twelve-prefill prototype on every shared case."
+  }
+}