| """ |
| Semantic Change Search Engine — Gradio frontend (rewired for real change |
| retrieval). |
| |
| A natural-language query is scored against every bi-temporal pair's *change* |
| representation (not single images), returning a ranked list of change events. |
| Each result shows the actual T1/T2 tiles of the matched pair, a |
| query-conditioned change heatmap (per-patch Δ-similarity T1→T2) on the After |
| tile, a relative match score, and a seasonal-vs-permanent note grounded in the dataset's |
| labels. |
| |
| Selectors: Dataset, Encoder (CLIP / GeoRSCLIP / RemoteCLIP), Approach |
| (naive / zero_shot Δ-similarity / patch (localised per-patch Δ) / peft adapter). |
| |
| CLI: |
| python -m src.app --dataset dynamic_earthnet --root data/DynamicEarthNet \ |
| --encoder clip_vitl14 --approach zero_shot |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import csv |
| import os |
| import tempfile |
| import traceback |
| from dataclasses import dataclass, field |
| from pathlib import Path |
| from typing import List, Optional |
|
|
| import numpy as np |
| from PIL import Image |
|
|
| from src.embeddings import load_or_compute |
| from src.encoders import get_encoder |
| from src.retrieval import APPROACHES, ChangeRetriever |
| from src.heatmap import generate_change_heatmap, generate_heatmap |
| from src.rerank import RERANK_STRATEGIES |
|
|
| _PROJECT_ROOT = Path(__file__).resolve().parent.parent |
| _SNOW = "snow_and_ice" |
|
|
| _CSV_HEADER = ["rank", "location", "before", "after", "score", |
| "match_0_1", "caption", "land_cover_change"] |
|
|
|
|
| _EXPORT_DIR = None |
|
|
|
|
| def _safe_name(raw: str) -> str: |
| return "".join(c if (c.isalnum() or c in "._-") else "_" for c in str(raw)) or "image" |
|
|
|
|
| def _export_dir() -> str: |
| """One reused temp dir for every app export (named images + result CSV). With |
| deterministic basenames the files overwrite in place, so the footprint stays |
| bounded across a long-running session instead of leaking a dir per query.""" |
| global _EXPORT_DIR |
| if _EXPORT_DIR is None: |
| _EXPORT_DIR = tempfile.mkdtemp(prefix="change_app_") |
| return _EXPORT_DIR |
|
|
|
|
| def materialize_image(img, raw_name: str) -> Optional[str]: |
| """Save a PIL image to a PNG with a meaningful basename and return its path. |
| |
| Gradio names a downloaded image after the served file's basename, so passing a |
| named path (rather than a bare PIL object, which downloads as ``image.png``) |
| gives the user a sensible filename like ``after_2065_2018-07-01.png``. |
| Returns None when *img* is None. |
| """ |
| if img is None: |
| return None |
| path = os.path.join(_export_dir(), _safe_name(raw_name) + ".png") |
| img.save(path) |
| return path |
|
|
|
|
| def results_to_csv(rows, dataset: str) -> Optional[str]: |
| """Write ranked result *rows* to a CSV with a clean, dataset-named basename |
| (e.g. ``change_results_levir_mci.csv``) so the in-app download has a usable |
| name + extension, not a random temp name. Returns the path, or None if no rows. |
| """ |
| if not rows: |
| return None |
| safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in dataset) or "results" |
| path = os.path.join(_export_dir(), f"change_results_{safe}.csv") |
| with open(path, "w", newline="", encoding="utf-8") as fh: |
| writer = csv.writer(fh) |
| writer.writerow(_CSV_HEADER) |
| writer.writerows(rows) |
| return path |
|
|
|
|
| def displayable_events(evs): |
| """Split ranked events for the results view. |
| |
| Returns ``(shown, top)`` where ``shown`` is the events that have a displayable |
| image (``heatmap`` or ``t2_img`` — the all-matches tiles + View buttons index |
| these 1:1) and ``top`` is the Top-match event: the first displayable one, |
| falling back to ``evs[0]`` only if none are displayable (``None`` if empty). |
| This keeps the Top-match panel and the first View button pointing at the same |
| event even when a higher-ranked pair's images fail to load. |
| """ |
| shown = [e for e in evs if (e.heatmap or e.t2_img) is not None] |
| top = shown[0] if shown else (evs[0] if evs else None) |
| return shown, top |
|
|
|
|
| def _app_datasets() -> list: |
| """Datasets offered in the app UI: those with a registered query set, so |
| label-grounded retrieval, relevance and the seasonal note all work. Any |
| loader without a query set (no labels / no benchmarked data) is excluded, |
| since it would otherwise error if picked. |
| """ |
| from src.queries import get_queries |
| from src.datasets.registry import list_datasets |
| return [d for d in list_datasets() if get_queries(d)] |
|
|
|
|
| |
| |
| |
| DATASET_LABELS = { |
| "dynamic_earthnet": "Dynamic EarthNet — land-cover transitions", |
| "levir_cc": "LEVIR-CC — building/road (captions)", |
| "levir_mci": "LEVIR-CC — building/road change", |
| "second_cc": "SECOND-CC — 6-class land cover", |
| "qfabric_teo": "QFabric — change type", |
| "qfabric_status": "QFabric — construction status (weak)", |
| } |
| ENCODER_LABELS = { |
| "clip_vitl14": "CLIP ViT-L/14", |
| "georsclip": "GeoRSCLIP", |
| "remoteclip": "RemoteCLIP", |
| } |
|
|
|
|
| def _labeled(choices, labels): |
| """``[(display_label, value)]`` for a Dropdown; value stays the registry key.""" |
| return [(labels.get(c, c), c) for c in choices] |
|
|
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| _QFABRIC_CROPS = str(_PROJECT_ROOT / "data" / "QFabric" / "teochat_crops") |
| DATASET_PROFILES = { |
| "dynamic_earthnet": {"root": str(_PROJECT_ROOT / "data" / "DynamicEarthNet"), |
| "split": "test", "pairing": "bimonthly"}, |
| |
| |
| |
| "levir_mci": {"root": str(_PROJECT_ROOT / "data" / "_levir_mci" / "extracted" / "LEVIR-MCI-dataset"), |
| "split": "test", "color_mode": "rgb"}, |
| "second_cc": {"root": str(_PROJECT_ROOT / "data" / "_second_cc" / "extracted" / "SECOND-CC-AUG"), |
| "split": "test", "color_mode": "rgb"}, |
| "qfabric_teo": {"root": _QFABRIC_CROPS, "split": None, "color_mode": "rgb", |
| "loader_extra": {"labels_path": str(_PROJECT_ROOT / "data" / "QFabric" |
| / "qfabric_teo_labels.json"), |
| "max_per_class": 120}}, |
| "qfabric_status": {"root": _QFABRIC_CROPS, "split": None, "color_mode": "rgb", |
| "loader_extra": {"labels_path": str(_PROJECT_ROOT / "data" / "QFabric" |
| / "qfabric_status_labels.json"), |
| "max_per_class": 120}}, |
| } |
|
|
| |
| |
| |
| |
| DATASET_RANK = { |
| "levir_mci": 0.83, "second_cc": 0.70, |
| "qfabric_teo": 0.27, "dynamic_earthnet": 0.19, "qfabric_status": 0.08, |
| } |
|
|
|
|
| def _app_dataset_choices() -> list: |
| """Datasets offered in the app dropdown: those with a query set AND a launch |
| profile (so switching to them in the UI actually loads them), sorted by best |
| achieved result (descending) so the strongest corpora appear first.""" |
| profiled = [d for d in _app_datasets() if d in DATASET_PROFILES] |
| return sorted(profiled, key=lambda d: -DATASET_RANK.get(d, 0.0)) |
|
|
|
|
| @dataclass |
| class RunConfig: |
| |
| |
| |
| |
| |
| dataset: str = "levir_mci" |
| encoder: str = "georsclip" |
| approach: str = "zero_shot" |
| root: str = str(_PROJECT_ROOT / "data" / "_levir_mci" / "extracted" / "LEVIR-MCI-dataset") |
| pairing: str = "bimonthly" |
| split: Optional[str] = "test" |
| cache_dir: str = str(_PROJECT_ROOT / "data" / "cache") |
| feature_mode: str = "difference" |
| top_k: int = 5 |
| |
| color_mode: str = "rgb" |
| use_lora: bool = False |
| |
| geo_filter: bool = False |
| rerank: bool = False |
| rerank_strategy: str = "diversity" |
| |
| |
| loader_extra: dict = field(default_factory=dict) |
|
|
|
|
| @dataclass |
| class ChangeEvent: |
| rank: int |
| pair_id: str |
| location: str |
| t1_key: str |
| t2_key: str |
| score: float |
| confidence: float |
| caption: str |
| seasonal_note: str |
| t1_img: Optional[Image.Image] |
| t2_img: Optional[Image.Image] |
| heatmap: Optional[Image.Image] |
|
|
|
|
| class SemanticChangeSearch: |
| _CSS = """ |
| .top-card {border: 1px solid var(--border-color-primary, #d0d0d0); |
| border-radius: 12px; padding: 14px 18px; |
| background: var(--background-fill-secondary, #fafafa);} |
| /* Single-hue intensity ramp, NOT a traffic light: the pill shows a |
| relative match rank within the candidate set, not absolute quality, |
| so deep->pale blue = stronger->weaker *position* (no green=good / |
| red=bad signal that would oversell a weak retrieval). Hue matches |
| the #1565c0 stats accent. */ |
| .conf-pill {display:inline-block; padding:4px 12px; border-radius:999px; |
| background:#1565c0; color:white; font-weight:600;} |
| .conf-pill.mid {background:#6fa8dc; color:#10243a;} |
| .conf-pill.low {background:#cfe0f2; color:#10243a;} |
| #search-btn { min-height: 3.5rem !important; font-size: 1.1rem !important; |
| letter-spacing: 0.03em; } |
| .stats-card {border:1px solid var(--border-color-primary,#d0d0d0); |
| border-left:4px solid #1565c0; |
| border-radius:10px; padding:10px 14px; |
| background:var(--background-fill-secondary,#f5f9ff); |
| font-size:0.95em; line-height:1.7;} |
| .stats-card code {background:rgba(21,101,192,0.08); padding:1px 6px; |
| border-radius:4px; font-weight:600;} |
| .stats-err {border-left-color:#c62828; background:#fff4f4;} |
| /* Keep each comparison slider sized to its (square) tile so the swipe |
| divider stays bounded to the image instead of travelling into empty |
| letterbox margins. */ |
| .bounded-slider {max-width: 360px !important; margin-left:auto; margin-right:auto;} |
| /* Per-image download buttons: bigger, bold, each its own colour-coded box |
| so they read as distinct, obvious download actions (not plain links). */ |
| .dl-row {gap: 12px;} |
| /* elem_classes land on the button element itself, so target both the class |
| and a nested button to be robust across Gradio versions. */ |
| .dl-btn, .dl-btn button {font-size: 1.02rem !important; font-weight: 700 !important; |
| border-radius: 10px !important; padding: 12px 10px !important; |
| border-width: 2px !important; border-style: solid !important;} |
| .dl-before, .dl-before button {background:#1565c0 !important; border-color:#0d47a1 !important; color:#fff !important;} |
| .dl-after, .dl-after button {background:#1f9d76 !important; border-color:#14785a !important; color:#fff !important;} |
| .dl-heat, .dl-heat button {background:#e0822e !important; border-color:#b5641a !important; color:#fff !important;} |
| /* All-matches grid: each "View" button spans its tile's width. */ |
| .view-btn, .view-btn button {width:100% !important; font-weight:600 !important;} |
| /* Tiles are display-only — the View button below is the click target, so |
| don't imply the image itself is clickable. */ |
| .match-tile, .match-tile * {cursor: default !important;} |
| """ |
|
|
| def __init__(self, cfg: RunConfig): |
| self.cfg = cfg |
| self._build(cfg) |
|
|
| |
| @classmethod |
| def from_components(cls, dataset, encoder, store, cfg: RunConfig, |
| adapter=None) -> "SemanticChangeSearch": |
| self = cls.__new__(cls) |
| self.cfg = cfg |
| self.dataset = dataset |
| self.encoder = encoder |
| self.store = store |
| self.retriever = ChangeRetriever(store, encoder, |
| feature_mode=cfg.feature_mode) |
| self._adapter_feature_mode = cfg.feature_mode |
| if adapter is not None: |
| self.retriever.set_adapter(adapter, cfg.feature_mode) |
| self._adapter = adapter |
| self._patch_t1 = self._patch_t2 = None |
| self._init_extensions(cfg) |
| return self |
|
|
| def _build(self, cfg: RunConfig) -> None: |
| from src.datasets.registry import build_dataset |
| from src.embeddings import cache_path as _cache_path |
| from src.embeddings import cache_tag_for |
| self.dataset = build_dataset(cfg.dataset, root=cfg.root, |
| pairing=cfg.pairing, split=cfg.split, |
| color_mode=cfg.color_mode, |
| **cfg.loader_extra) |
| self.encoder = get_encoder(cfg.encoder) |
|
|
| |
| |
| split_str = cfg.split or "all" |
| cache_tag = cache_tag_for(split_str, cfg.color_mode, cfg.use_lora) |
|
|
| if cfg.use_lora: |
| lora_cache = _cache_path(cfg.cache_dir, self.dataset.name, |
| self.encoder.name, tag=cache_tag) |
| if not lora_cache.exists(): |
| raise RuntimeError( |
| f"LoRA embedding cache not found: {lora_cache.name}\n" |
| "Train LoRA first: python -m scripts.run_pipeline --lora ...\n" |
| "Then re-open the app with 'Use LoRA embeddings' enabled." |
| ) |
|
|
| self.store = load_or_compute(self.dataset, self.encoder, |
| cache_dir=cfg.cache_dir, |
| cache_tag=cache_tag) |
| self.retriever = ChangeRetriever(self.store, self.encoder, |
| feature_mode=cfg.feature_mode) |
| self._adapter = self._maybe_load_adapter(cfg) |
| if self._adapter is not None: |
| self.retriever.set_adapter(self._adapter, self._adapter_feature_mode) |
| self._patch_t1 = self._patch_t2 = None |
| self._init_extensions(cfg) |
|
|
| def _init_extensions(self, cfg: RunConfig) -> None: |
| """Load geo-filter and reranker if aoi_metadata.json is present.""" |
| meta_path = Path(cfg.root) / "aoi_metadata.json" |
| if meta_path.exists(): |
| from src.geo_filter import GeoFilter |
| from src.rerank import Reranker |
| self._geo_filter = GeoFilter(meta_path) |
| self._reranker = Reranker(meta_path) |
| else: |
| self._geo_filter = None |
| self._reranker = None |
|
|
| def _maybe_load_adapter(self, cfg: RunConfig): |
| |
| |
| |
| |
| from src.model import adapter_path |
| fname = adapter_path(cfg.dataset, cfg.encoder, cfg.color_mode).name |
| path = Path(cfg.cache_dir).parent / "models" / fname |
| if not path.exists(): |
| path = _PROJECT_ROOT / "models" / fname |
| if path.exists(): |
| try: |
| from src.model import load_adapter |
| adapter, meta = load_adapter(str(path)) |
| |
| |
| |
| self._adapter_feature_mode = meta.get("feature_mode", cfg.feature_mode) |
| print(f"Loaded PEFT adapter: {path} (feature_mode={self._adapter_feature_mode})") |
| return adapter |
| except Exception as exc: |
| print(f"Adapter load failed ({exc}); PEFT disabled.") |
| self._adapter_feature_mode = cfg.feature_mode |
| return None |
|
|
| |
| def _patch_scores(self, text: str) -> np.ndarray: |
| """Localised patch-level Δ-similarity (REPORT Appendix B.10, best DEN |
| config). Per-patch embeddings for the whole corpus are loaded from the |
| on-disk patch cache when warm (instant first query) and only computed + |
| cached on a miss; the result is held on the engine for later queries. |
| The patch rows align with ``self.store.pairs`` (same pair list as the |
| pair store), so ``query()`` can index ``_patch_t1[i]`` by store index.""" |
| from src.benchmark import encode_query |
| from src.embeddings import cache_tag_for, load_or_compute_patches |
| from src.retrieval import top_patch_change_scores |
| if getattr(self, "_patch_t1", None) is None: |
| split_str = self.cfg.split or "all" |
| |
| |
| |
| cache_tag = cache_tag_for(split_str, self.cfg.color_mode) |
| patch_store = load_or_compute_patches( |
| self.dataset, self.encoder, self.store.pairs, |
| cache_dir=self.cfg.cache_dir, cache_tag=cache_tag, |
| batch_size=32, progress=True, |
| ) |
| self._patch_t1 = patch_store.patch_t1 |
| self._patch_t2 = patch_store.patch_t2 |
| t = encode_query(self.encoder, text) |
| return top_patch_change_scores(self._patch_t1, self._patch_t2, t) |
|
|
| |
| def query( |
| self, |
| text: str, |
| approach: str, |
| top_k: int, |
| *, |
| geo_region: str = "All", |
| rerank_strategy: Optional[str] = None, |
| ) -> List[ChangeEvent]: |
| """Retrieve top-K change events matching *text*. |
| |
| Parameters |
| ---------- |
| text: Natural-language change query. |
| approach: ``"naive"``, ``"zero_shot"``, or ``"peft"``. |
| top_k: Number of results to return. |
| geo_region: If not ``"All"``, restrict pairs to that region |
| (requires ``aoi_metadata.json`` to have been loaded). |
| rerank_strategy: ``"diversity"``, ``"coherence"``, or ``None`` |
| (disabled). Requires ``aoi_metadata.json``. |
| """ |
| if approach == "patch": |
| scores = self._patch_scores(text) |
| else: |
| if approach == "peft" and self.retriever.adapter is None: |
| raise RuntimeError( |
| "PEFT selected but no adapter found. Train one with " |
| "`python -m src.train` or pick zero_shot.") |
| scores = self.retriever.score_all(text, approach=approach) |
|
|
| |
| if geo_region not in ("All", "", None) and self._geo_filter is not None: |
| allowed = { |
| p.location_id |
| for p in self._geo_filter.filter_by_region(self.store.pairs, geo_region) |
| } |
| mask = np.array([p.location_id in allowed for p in self.store.pairs]) |
| scores = np.where(mask, scores, -np.inf) |
|
|
| |
| finite = scores[np.isfinite(scores)] |
| lo = float(finite.min()) if len(finite) else 0.0 |
| hi = float(finite.max()) if len(finite) else 1.0 |
|
|
| |
| if rerank_strategy is not None and self._reranker is not None: |
| order = self._reranker.rerank( |
| scores, self.store.pairs, top_k, strategy=rerank_strategy |
| ) |
| else: |
| order = np.argsort(-scores, kind="stable")[:top_k] |
|
|
| |
| |
| |
| patches_cached = (approach == "patch" |
| and getattr(self, "_patch_t1", None) is not None) |
|
|
| events: List[ChangeEvent] = [] |
| for rank, i in enumerate(order): |
| pair = self.store.pairs[i] |
| t1 = t2 = hm = None |
| try: |
| t1, t2 = self.dataset.load_pair_images(pair) |
| except Exception as exc: |
| print(f"image load failed for {pair}: {exc}") |
| if t1 is not None and t2 is not None: |
| try: |
| |
| |
| pp1 = self._patch_t1[i] if patches_cached else None |
| pp2 = self._patch_t2[i] if patches_cached else None |
| _, hm = generate_change_heatmap(np.array(t1), np.array(t2), |
| text, self.encoder, alpha=0.5, |
| precomputed_p1=pp1, precomputed_p2=pp2) |
| if hm is None: |
| _, hm = generate_heatmap(np.array(t1), np.array(t2), |
| text, self.encoder, alpha=0.5) |
| except Exception as exc: |
| print(f"heatmap failed: {exc}") |
| conf = (float(scores[i]) - lo) / (hi - lo) if hi > lo else 1.0 |
| caption, note = self._describe(pair) |
| events.append(ChangeEvent( |
| rank=rank + 1, |
| pair_id=f"{pair.location_id}_{pair.t1_key}_{pair.t2_key}", |
| location=pair.location_id, t1_key=pair.t1_key, |
| t2_key=pair.t2_key, score=float(scores[i]), |
| confidence=round(conf, 3), caption=caption, |
| seasonal_note=note, t1_img=t1, t2_img=t2, heatmap=hm, |
| )) |
| return events |
|
|
| def _describe(self, pair) -> tuple[str, str]: |
| try: |
| label = self.dataset.get_pair_label(pair) |
| except Exception: |
| label = None |
| cap_fn = getattr(self.dataset, "text_caption_for_pair", None) |
| caption = cap_fn(pair) if cap_fn else ( |
| label.change_type if label else "unknown change") |
| if label is None: |
| return caption, "no label (unlabeled dataset)" |
| if _SNOW in (label.dominant_t1_class, label.dominant_t2_class): |
| return caption, "involves snow/ice - likely SEASONAL, not permanent" |
| if label.stable: |
| return caption, "labelled stable - weak/no change" |
| return caption, "permanent land-cover change" |
|
|
| |
| def stats_markdown(self) -> str: |
| """One-line corpus stats card (rendered as Gradio markdown).""" |
| n_pairs = len(self.store) |
| n_locs = len(self.dataset.list_locations()) |
| per_loc = (n_pairs / n_locs) if n_locs else 0 |
| adapter = "loaded" if self.retriever.adapter is not None else "none" |
| geo_status = "ready" if self._geo_filter is not None else "N/A (no metadata)" |
| rerank_status = "ready" if self._reranker is not None else "N/A (no metadata)" |
| return ( |
| f"<div class='stats-card'>" |
| f"<b>Dataset:</b> <code>{self.cfg.dataset}</code> | " |
| f"<b>Split:</b> <code>{self.cfg.split or 'all'}</code> | " |
| f"<b>Pairing:</b> <code>{self.cfg.pairing}</code> | " |
| f"<b>Color:</b> <code>{self.cfg.color_mode}</code> | " |
| f"<b>Locations:</b> <code>{n_locs}</code> | " |
| f"<b>Image pairs:</b> <code>{n_pairs}</code> " |
| f"({per_loc:.0f}/location) | " |
| f"<b>Encoder:</b> <code>{self.encoder.name}</code> " |
| f"({self.encoder.embed_dim}-d) | " |
| f"<b>PEFT adapter:</b> <code>{adapter}</code> | " |
| f"<b>Geo filter:</b> <code>{geo_status}</code> | " |
| f"<b>Re-ranking:</b> <code>{rerank_status}</code>" |
| f"</div>" |
| ) |
|
|
| def reload(self, dataset: str, encoder: str, approach: str, |
| color_mode: str = "rgb", use_lora: bool = False): |
| try: |
| |
| |
| |
| prof = DATASET_PROFILES.get(dataset, {}) |
| prof_root = prof.get("root") |
| |
| |
| |
| root = prof_root if prof_root and Path(prof_root).exists() else self.cfg.root |
| split = prof.get("split", self.cfg.split) |
| pairing = prof.get("pairing", self.cfg.pairing) |
| color = prof.get("color_mode", color_mode) |
| self.cfg = RunConfig( |
| dataset=dataset, encoder=encoder, approach=approach, |
| root=root, pairing=pairing, |
| split=split, cache_dir=self.cfg.cache_dir, |
| color_mode=color, use_lora=use_lora, |
| geo_filter=self.cfg.geo_filter, rerank=self.cfg.rerank, |
| rerank_strategy=self.cfg.rerank_strategy, |
| loader_extra=prof.get("loader_extra", {}), |
| ) |
| self._build(self.cfg) |
| lora_note = " + LoRA" if use_lora else "" |
| status = (f"Loaded {dataset} + {encoder}{lora_note} | " |
| f"color={self.cfg.color_mode} | approach={approach} | " |
| f"{len(self.store)} pairs") |
| return status, self.stats_markdown() |
| except Exception as exc: |
| traceback.print_exc() |
| return f"Error: {exc}", "<div class='stats-card stats-err'>Error</div>" |
|
|
| def build_interface(self): |
| import gradio as gr |
| engine = self |
| from src.encoders import list_encoders |
|
|
| |
| |
| APPROACH_HELP = "How each pair is scored. See **About** for what each option means." |
| ENCODER_HELP = ( |
| "clip_vitl14 (general purpose, 768-d) · georsclip (RS-pretrained ViT-B/32, 512-d) " |
| "· remoteclip (RS-pretrained ViT-L/14, 768-d). Weights download on first use." |
| ) |
| DATASET_HELP = ( |
| "Pair corpus to search over. Switching rebuilds (or loads cached) embeddings — " |
| "press Apply Settings afterwards." |
| ) |
| TOPK_HELP = "How many ranked change events to return." |
| QUERY_HELP = ("Describe the land-cover change you are looking for. " |
| "Specific transitions (e.g. 'forest cleared to bare soil') work best.") |
| COLOR_MODE_HELP = ( |
| "rgb = standard optical (R-G-B) · nrg = NIR-Red-Green false colour " |
| "(best zero-shot with GeoRSCLIP) · ndvi = vegetation index (single channel × 3). " |
| "NRG/NDVI need the NIR band and apply to Dynamic EarthNet only — other corpora " |
| "are RGB-only (nrg/ndvi are greyed out and ignored). Changing requires Apply Settings." |
| ) |
| LORA_HELP = ( |
| "Load LoRA-adapted embeddings pre-cached by run_pipeline --lora. " |
| "Cache must exist for the selected encoder + color mode. " |
| "Changing requires Apply Settings." |
| ) |
|
|
| with gr.Blocks(title="Open Vocabulary Temporal Change Retrieval", |
| analytics_enabled=False) as demo: |
| gr.Markdown( |
| "# Open Vocabulary Temporal Change Retrieval\n" |
| "Describe a land-cover change in plain language — the semantic change search " |
| "engine finds the satellite image pairs (and the timestep) where it happened. " |
| "*Research demo — results are approximate (see **About** below).*" |
| ) |
|
|
| with gr.Accordion("About / How it works", open=False): |
| gr.Markdown( |
| "**What it is.** A frozen vision–language model (CLIP / GeoRSCLIP / " |
| "RemoteCLIP) encodes each timestep of a bi-temporal satellite pair; the " |
| "**change** between them is matched against your text query by cosine " |
| "similarity. No per-query training — it is *open-vocabulary*.\n\n" |
| "**Open-vocabulary** means you can search for *any* change in free text — the " |
| "engine is never trained on the specific change types of a corpus. That is the " |
| "advantage over supervised models; the trade-off is lower absolute accuracy.\n\n" |
| "**Honest expectations.** This is a research demo, not a product. With frozen " |
| "encoders, open-vocabulary change retrieval is **weak in absolute terms** — the " |
| "best honestly-audited configuration reaches only **≈ 0.20 mAP** (5-fold " |
| "cross-validated), and recovery scales with how *visually salient* the change " |
| "is. Use the curated example queries for results that actually work.\n\n" |
| "**Approaches.** *Naive* = cos(text, After) — an image-retrieval baseline. " |
| "*Zero-shot* = cos(text, After) − cos(text, Before) — the temporal Δ, no " |
| "training. *Patch / localised* = top-3 per-patch Δ — it averages the three " |
| "most-changed patches, catching small localised change a whole-image embedding " |
| "would wash out; the best configuration on Dynamic EarthNet. *PEFT adapter* = a " |
| "small trained projection head (loaded only if a matching adapter exists).\n\n" |
| "**On PEFT/LoRA (honest note).** Light fine-tuning was trained and evaluated on " |
| "Dynamic EarthNet and QFabric only; on held-out data it does **not** beat the " |
| "frozen zero-shot encoders (it overfits the training scenes). Frozen " |
| "encoders + NRG + patch scoring is the strongest honestly-audited setup.\n\n" |
| "**Reading the results.** The **match score (0–1)** is a *relative* rank within " |
| "the returned set (min–max normalised), **not** a calibrated probability. The " |
| "**change heatmap** (jet) overlays the After image: warm = where the query's " |
| "presence grew most from Before→After; cool = little/no change. The **land-cover " |
| "change note** classes each result from its dataset label: *permanent* land-cover " |
| "change, *likely seasonal* (e.g. snow/ice, which recurs annually), or *stable* " |
| "(labelled weak/no change); *no label* on unlabelled corpora.\n\n" |
| "**Corpora (Settings → pick → Apply), sorted by best result:** LEVIR-CC " |
| "(building/road, default — strongest), SECOND-CC (six land-cover classes), " |
| "QFabric change-type (construction), Dynamic EarthNet (the report's primary " |
| "analysed corpus; subtle spectral change, weakest absolute results), and QFabric " |
| "construction-status (a distinct task, but retrieval is ≈ random — included for " |
| "completeness). Switching reloads embeddings, so press **Apply Settings**.\n\n" |
| "**Note on QFabric here:** this is the reduced *2-date* TEOChatlas crop subset " |
| "(change-type / status retrieval only). The full *5-date* QFabric with polygon " |
| "change-masks (for temporal pinpointing + pixel localization) is a future-work " |
| "direction that was dropped from scope (the dataset source is access-gated) — it " |
| "is **not** the same as the 2-date subset shown here." |
| ) |
|
|
| |
| with gr.Row(): |
| q = gr.Textbox( |
| label="Change query", scale=5, |
| value="new buildings or houses constructed", |
| placeholder="e.g. a new road or street; a new water body", |
| info=QUERY_HELP, |
| ) |
| a_dd = gr.Dropdown( |
| choices=[ |
| ("Naive — match the After image", "naive"), |
| ("Zero-shot — temporal Δ (no training)", "zero_shot"), |
| ("Patch — localised Δ (best on DEN)", "patch"), |
| ("PEFT adapter — trained head", "peft"), |
| ], |
| value=engine.cfg.approach, |
| label="Approach", |
| info=APPROACH_HELP, |
| scale=2, |
| ) |
| k = gr.Slider(1, 10, value=engine.cfg.top_k, step=1, |
| label="Top-K", info=TOPK_HELP, scale=1) |
| go = gr.Button("Search", variant="primary", size="lg", |
| elem_id="search-btn") |
|
|
| |
| |
| |
| |
| gr.Markdown( |
| "**Example queries — click to fill.** *Work best on the default LEVIR-CC corpus; " |
| "see About for per-corpus tips.*" |
| ) |
| with gr.Row(): |
| for _ex in [ |
| "new buildings or houses constructed", |
| "a new road or street built", |
| "a new water body or flooding", |
| "trees or vegetation cleared or grown", |
| ]: |
| gr.Button(_ex, size="sm").click(lambda v=_ex: v, None, q) |
|
|
| |
| _geo_available = engine._geo_filter is not None |
| _regions = engine._geo_filter.regions if _geo_available else ["All"] |
| _rerank_available = engine._reranker is not None |
|
|
| with gr.Accordion("Settings — corpus · model · filters", open=False): |
| stats_md = gr.Markdown(engine.stats_markdown()) |
|
|
| gr.Markdown( |
| "**Corpus & model** — pick the dataset, encoder, colour mode or adapter, " |
| "then press **Apply settings** to (re)load embeddings." |
| ) |
| with gr.Row(): |
| d_dd = gr.Dropdown(_labeled(_app_dataset_choices(), DATASET_LABELS), |
| value=engine.cfg.dataset, |
| label="Dataset", info=DATASET_HELP) |
| e_dd = gr.Dropdown(_labeled(list_encoders(), ENCODER_LABELS), |
| value=engine.cfg.encoder, |
| label="Encoder", info=ENCODER_HELP) |
| with gr.Row(): |
| color_dd = gr.Dropdown( |
| ["rgb", "nrg", "ndvi"], value=engine.cfg.color_mode, |
| label="Color mode", info=COLOR_MODE_HELP, |
| |
| interactive=(engine.cfg.dataset == "dynamic_earthnet"), |
| ) |
| lora_chk = gr.Checkbox( |
| label="Use LoRA embeddings", |
| value=engine.cfg.use_lora, |
| info=LORA_HELP, |
| ) |
| with gr.Row(): |
| apply = gr.Button("Apply settings", variant="secondary") |
| status = gr.Textbox( |
| label="Engine status", interactive=False, scale=4, |
| value=f"{engine.cfg.dataset} + {engine.cfg.encoder} | " |
| f"color={engine.cfg.color_mode} | " |
| f"{len(engine.store)} pairs | " |
| f"approach={engine.cfg.approach}") |
|
|
| gr.Markdown( |
| "---\n**Filters & re-ranking** — applied on the **next Search** (no Apply " |
| "needed). Both require `aoi_metadata.json` in the dataset root (present for " |
| "Dynamic EarthNet); otherwise they are greyed out." |
| ) |
| with gr.Row(): |
| geo_chk = gr.Checkbox( |
| label="Geographic filter", |
| value=engine.cfg.geo_filter and _geo_available, |
| interactive=_geo_available, |
| info="Restrict results to one continental region." |
| if _geo_available |
| else "Requires aoi_metadata.json in the dataset root.", |
| ) |
| geo_dd = gr.Dropdown( |
| _regions, value="All", label="Region", |
| interactive=_geo_available, |
| ) |
| with gr.Row(): |
| rerank_chk = gr.Checkbox( |
| label="Re-rank results", |
| value=engine.cfg.rerank and _rerank_available, |
| interactive=_rerank_available, |
| info="Post-process ranking for spatial quality." |
| if _rerank_available |
| else "Requires aoi_metadata.json in the dataset root.", |
| ) |
| rerank_dd = gr.Dropdown( |
| list(RERANK_STRATEGIES), |
| value=engine.cfg.rerank_strategy, |
| label="Strategy", |
| interactive=_rerank_available, |
| info=( |
| "diversity = prefer unique locations per result · " |
| "coherence = cluster near the top-1 location" |
| ), |
| ) |
|
|
| |
| |
| |
| |
| def _color_for_dataset(dataset: str): |
| if dataset == "dynamic_earthnet": |
| return gr.update(interactive=True) |
| return gr.update(value="rgb", interactive=False) |
|
|
| d_dd.change(_color_for_dataset, d_dd, color_dd) |
|
|
| apply.click(engine.reload, |
| [d_dd, e_dd, a_dd, color_dd, lora_chk], |
| [status, stats_md]) |
|
|
| |
| gr.Markdown("## Top match") |
| |
| |
| events_state = gr.State([]) |
| with gr.Group(elem_classes="top-card"): |
| with gr.Row(): |
| |
| |
| |
| |
| |
| |
| |
| cmp = gr.ImageSlider( |
| label="Before ↔ After — drag the divider to compare", |
| height=340, width=340, interactive=False, format="png", |
| buttons=[], elem_classes="bounded-slider", |
| ) |
| |
| |
| |
| hm_cmp = gr.ImageSlider( |
| label="After ↔ change heatmap — drag to reveal Δ", |
| height=340, width=340, interactive=False, format="png", |
| buttons=[], elem_classes="bounded-slider", |
| ) |
| gr.Markdown( |
| "_Left: swipe **Before ↔ After**. Right: swipe the **After** image against the " |
| "query-conditioned **change heatmap** (jet — warm = query-change grew, cool = " |
| "little/none) to dial the overlay in and out. See About._" |
| ) |
| summary = gr.Markdown("*Press Search to retrieve.*") |
| |
| |
| |
| gr.Markdown("**Download images**") |
| with gr.Row(elem_classes="dl-row"): |
| dl_before = gr.DownloadButton( |
| "Download Before", visible=False, size="lg", |
| elem_classes=["dl-btn", "dl-before"]) |
| dl_after = gr.DownloadButton( |
| "Download After", visible=False, size="lg", |
| elem_classes=["dl-btn", "dl-after"]) |
| dl_heat = gr.DownloadButton( |
| "Download heatmap", visible=False, size="lg", |
| elem_classes=["dl-btn", "dl-heat"]) |
|
|
| |
| |
| |
| |
| |
| |
| gr.Markdown("## All matches") |
| MAX_RESULTS = 10 |
| tiles, view_btns = [], [] |
| with gr.Column(elem_classes="matches-grid"): |
| for _r in range(2): |
| with gr.Row(): |
| for _c in range(5): |
| with gr.Column(min_width=140): |
| tiles.append(gr.Image( |
| visible=False, interactive=False, height=220, |
| show_label=False, format="png", buttons=[], |
| elem_classes="match-tile")) |
| view_btns.append(gr.Button( |
| "View", visible=False, size="sm", |
| elem_classes="view-btn")) |
|
|
| with gr.Accordion("Results table", open=True): |
| table = gr.Dataframe( |
| headers=["rank", "location", "Before", "After", "score", |
| "match (0–1)", "caption", "land cover change"], |
| interactive=False, wrap=True, |
| ) |
| gr.Markdown( |
| "_Columns: **score** = raw change Δ-similarity (model units); " |
| "**match (0–1)** = relative rank within this query (min–max normalised, " |
| "not a probability); **caption** = the pair's dataset caption; " |
| "**land cover change** = permanent / likely-seasonal / stable note " |
| "(see About). Before/After are the two timesteps._" |
| ) |
|
|
| |
| |
| |
| dl = gr.File(label="Download ranked results (CSV)", visible=False, |
| interactive=False) |
|
|
| def _pill(c: float) -> str: |
| cls = "low" if c < 0.4 else ("mid" if c < 0.7 else "") |
| return (f"<span class='conf-pill {cls}' title='relative match score " |
| f"(min–max normalized over the candidate set), not a calibrated " |
| f"probability'>match {c:.2f}</span>") |
|
|
| def _event_md(e) -> str: |
| """The Top-match detail markdown for one change event.""" |
| return ( |
| f"### {e.caption}\n" |
| f"{_pill(e.confidence)} " |
| f"**Location** `{e.location}` " |
| f"**Before → After** `{e.t1_key}` → `{e.t2_key}` " |
| f"**Score** `{e.score:.4f}`\n\n" |
| f"_Reasoning:_ {e.seasonal_note}." |
| ) |
|
|
| def _dl(path): |
| return gr.update(value=path, visible=path is not None) |
|
|
| def _event_view(e): |
| """Materialise an event's images to named PNGs and return the two |
| swipe-slider tuples plus the three image paths (Before / After / |
| heatmap). Named paths give meaningful download filenames; a slider |
| tuple is None if a side is missing (None in a tuple breaks ImageSlider).""" |
| b = materialize_image(e.t1_img, f"before_{e.location}_{e.t1_key}") |
| a = materialize_image(e.t2_img, f"after_{e.location}_{e.t2_key}") |
| h = materialize_image( |
| e.heatmap, f"heatmap_{e.location}_{e.t1_key}_to_{e.t2_key}") |
| before_after = (b, a) if b and a else None |
| after_heat = (a, h) if a and h else None |
| return before_after, after_heat, b, a, h |
|
|
| def handle(text, approach, top_k, |
| geo_enabled, geo_region, rerank_enabled, rerank_strategy, |
| progress=gr.Progress()): |
| try: |
| progress(0.05, desc="Scoring corpus against your query… " |
| "(first query on a dataset/approach encodes it — a few seconds)") |
| active_geo = geo_region if geo_enabled else "All" |
| active_rerank = rerank_strategy if rerank_enabled else None |
| evs = engine.query( |
| text, approach, int(top_k), |
| geo_region=active_geo, |
| rerank_strategy=active_rerank, |
| ) |
| except Exception as exc: |
| return (None, None, ( |
| f"**Error:** {exc}\n\n*If you just changed Dataset / Encoder / Color mode, " |
| "press **Apply Settings** first. For PEFT, an adapter must exist for the " |
| "selected encoder + colour mode.*"), [], |
| gr.update(visible=False), gr.update(visible=False), |
| gr.update(visible=False), gr.update(visible=False), [], |
| *[gr.update(value=None, visible=False) for _ in range(MAX_RESULTS)], |
| *[gr.update(visible=False) for _ in range(MAX_RESULTS)]) |
| if not evs: |
| return (None, None, ( |
| "*No results for this query. Try a curated example above, or a different " |
| "**Approach** (patch / zero-shot).*"), [], |
| gr.update(visible=False), gr.update(visible=False), |
| gr.update(visible=False), gr.update(visible=False), [], |
| *[gr.update(value=None, visible=False) for _ in range(MAX_RESULTS)], |
| *[gr.update(visible=False) for _ in range(MAX_RESULTS)]) |
| progress(0.9, desc="Rendering results…") |
| rows = [[e.rank, e.location, e.t1_key, e.t2_key, |
| round(e.score, 4), e.confidence, e.caption, |
| e.seasonal_note] for e in evs] |
| |
| |
| |
| shown, top = displayable_events(evs) |
| tile_ups, btn_ups = [], [] |
| for i in range(MAX_RESULTS): |
| if i < len(shown): |
| e = shown[i] |
| kind = "heatmap" if e.heatmap is not None else "after" |
| tp = materialize_image( |
| e.heatmap or e.t2_img, |
| f"rank{e.rank}_{kind}_{e.location}_{e.t1_key}_to_{e.t2_key}") |
| tile_ups.append(gr.update(value=tp, visible=True)) |
| btn_ups.append(gr.update( |
| value=f"View #{e.rank} · {e.location}", visible=True)) |
| else: |
| tile_ups.append(gr.update(value=None, visible=False)) |
| btn_ups.append(gr.update(visible=False)) |
| before_after, after_heat, b, a, h = _event_view(top) |
| csv_path = results_to_csv(rows, engine.cfg.dataset) |
| return (before_after, after_heat, _event_md(top), rows, |
| _dl(csv_path), _dl(b), _dl(a), _dl(h), shown, |
| *tile_ups, *btn_ups) |
|
|
| def make_loader(i): |
| """A tile's View button → load result *i* into the Top-match view and |
| refresh the per-image download buttons (preview-free, always reliable).""" |
| def _load(shown): |
| if not shown or i >= len(shown): |
| return tuple(gr.update() for _ in range(6)) |
| e = shown[i] |
| before_after, after_heat, b, a, h = _event_view(e) |
| return before_after, after_heat, _event_md(e), _dl(b), _dl(a), _dl(h) |
| return _load |
|
|
| outputs = [cmp, hm_cmp, summary, table, dl, |
| dl_before, dl_after, dl_heat, events_state, *tiles, *view_btns] |
| inputs = [q, a_dd, k, geo_chk, geo_dd, rerank_chk, rerank_dd] |
| go.click(handle, inputs, outputs) |
| q.submit(handle, inputs, outputs) |
| for _i, _btn in enumerate(view_btns): |
| _btn.click(make_loader(_i), [events_state], |
| [cmp, hm_cmp, summary, dl_before, dl_after, dl_heat]) |
|
|
| |
| |
| |
| _APPROACHES = {"naive", "zero_shot", "patch", "peft"} |
|
|
| def _prefill_from_url(request: gr.Request = None): |
| params = dict(request.query_params or {}) if request is not None else {} |
| q_up = gr.update(value=params["q"]) if params.get("q") else gr.update() |
| a_val = params.get("approach") |
| a_up = gr.update(value=a_val) if a_val in _APPROACHES else gr.update() |
| k_up = gr.update() |
| try: |
| k_val = int(params.get("k", "")) |
| if 1 <= k_val <= 10: |
| k_up = gr.update(value=k_val) |
| except (TypeError, ValueError): |
| pass |
| return q_up, a_up, k_up |
|
|
| demo.load(_prefill_from_url, None, [q, a_dd, k]) |
| return demo |
|
|
|
|
| def parse_args(): |
| p = argparse.ArgumentParser(description="Semantic Change Search Engine") |
| from src.encoders import list_encoders |
| p.add_argument("--dataset", default="levir_mci", |
| choices=_app_dataset_choices()) |
| p.add_argument("--encoder", default="georsclip", |
| choices=list_encoders()) |
| p.add_argument("--approach", default="zero_shot", |
| choices=list(APPROACHES) + ["patch"]) |
| p.add_argument("--root", default=None, |
| help="dataset directory; default = the selected dataset's profile root") |
| p.add_argument("--pairing", default="bimonthly") |
| p.add_argument("--split", default=None, |
| help="split (train|val|test|all); default = the dataset's profile split") |
| p.add_argument("--cache-dir", default=str(_PROJECT_ROOT / "data" / "cache")) |
| p.add_argument("--port", type=int, default=7860) |
| p.add_argument("--host", default="127.0.0.1", |
| help="Bind address. Default 127.0.0.1 (local only). Use 0.0.0.0 to " |
| "expose on the LAN; on a HuggingFace Space 0.0.0.0 is auto-selected.") |
| |
| p.add_argument("--color-mode", default=None, choices=["rgb", "nrg", "ndvi"], |
| help="Image colour mode; default = the dataset's profile colour " |
| "(rgb for all but DEN, which defaults to nrg).") |
| p.add_argument("--lora", action="store_true", default=False, |
| help="Load LoRA-adapted embeddings (must be pre-cached by run_pipeline --lora). " |
| "Toggle in-app via Settings.") |
| p.add_argument("--no-lora", dest="lora", action="store_false") |
| |
| p.add_argument("--geo-filter", action="store_true", default=False, |
| help="Enable geographic region filter at startup (toggle in-app anytime).") |
| p.add_argument("--no-geo-filter", dest="geo_filter", action="store_false") |
| p.add_argument("--rerank", action="store_true", default=False, |
| help="Enable post-retrieval re-ranking at startup (toggle in-app anytime).") |
| p.add_argument("--no-rerank", dest="rerank", action="store_false") |
| p.add_argument("--rerank-strategy", default="diversity", |
| choices=list(RERANK_STRATEGIES), |
| help="Re-ranking strategy: diversity (default) or coherence.") |
| a = p.parse_args() |
| |
| |
| |
| |
| prof = DATASET_PROFILES.get(a.dataset, {}) |
| root = a.root or prof.get("root") or str(_PROJECT_ROOT / "data" / "DynamicEarthNet") |
| split = a.split if a.split is not None else prof.get("split", "test") |
| color = a.color_mode or prof.get("color_mode") or "nrg" |
| |
| |
| on_space = bool(os.environ.get("SPACE_ID") or os.environ.get("SPACE_HOST")) |
| server_name = "0.0.0.0" if on_space else a.host |
| |
| show_error = server_name in ("127.0.0.1", "localhost") |
| return RunConfig( |
| dataset=a.dataset, encoder=a.encoder, approach=a.approach, |
| root=root, pairing=a.pairing, |
| split=None if split == "all" else split, |
| cache_dir=a.cache_dir, |
| color_mode=color, |
| use_lora=a.lora, |
| geo_filter=a.geo_filter, |
| rerank=a.rerank, |
| rerank_strategy=a.rerank_strategy, |
| loader_extra=prof.get("loader_extra", {}), |
| ), a.port, server_name, show_error |
|
|
|
|
| def main(): |
| import gradio as gr |
| cfg, port, server_name, show_error = parse_args() |
| print(f"Starting engine: dataset={cfg.dataset} encoder={cfg.encoder} " |
| f"approach={cfg.approach} bind={server_name}:{port}") |
| engine = SemanticChangeSearch(cfg) |
| demo = engine.build_interface() |
| |
| theme = gr.themes.Ocean( |
| |
| |
| text_size=gr.themes.sizes.text_lg, |
| font=["system-ui", "-apple-system", "Segoe UI", "Arial", "sans-serif"], |
| font_mono=["Consolas", "Monaco", "monospace"], |
| ) |
| demo.launch(server_name=server_name, server_port=port, show_error=show_error, |
| theme=theme, css=SemanticChangeSearch._CSS) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|