| { |
| "model": "fusion-perception-1", |
| "version": "v0.1-preview", |
| "capability": "landmark / place retrieval", |
| "metric": "mAP in percent, revisited Oxford / Paris protocol (Radenovic et al., CVPR 2018), Medium and Hard", |
| "protocol_notes": { |
| "harness": "the official revisitop compute_map, vendored verbatim; a stage-0 audit scored our implementation against the file fetched byte-for-byte from github.com/filipradenovic/revisitop and found a maximum mAP difference of 0.0 across three descriptor sets on both datasets", |
| "queries": "70 per dataset, cropped to the ground-truth bounding box before embedding, as the official example evaluation does; gallery images are never cropped", |
| "gallery_no_distractors": { |
| "roxford5k": 4993, |
| "rparis6k": 6322 |
| }, |
| "gallery_plus_1m": { |
| "roxford5k": 1005994, |
| "rparis6k": 1007323 |
| }, |
| "distractor_set": "revisitop1m, 1001001 images over 100 archives", |
| "reranking": "AMES (Suma et al., ECCV 2024) over the authors' frozen DINOv2-B local descriptors and their published full-precision checkpoint, shortlist top-1600, ensemble score = 0.55 * global_cosine + 0.45 * sigmoid(0.3 * ames_logit). This is the AMES paper-default cell. Other lambda / temperature cells were measured as diagnostics and are not reported here.", |
| "shortlist": "junk-aware, as in the AMES and DELG evaluation code: ground-truth junk ids are moved behind the shortlist before reranking. Every published two-stage row in the comparison below does the same.", |
| "precision": "descriptors extracted in float16 on an A10G; head applied in float32", |
| "two_stage_reproduction": { |
| "script": "rerank.py in this repository", |
| "command": "python rerank.py --dataset {roxford5k,rparis6k} --ames-dir ./ames_assets --images-root <benchmark jpg dir> --head {standard,decon}", |
| "defaults": "--topk 1600 --lambdas 0.55 --temps 0.3, the reported cell", |
| "plus_1m": "add --distractor-locals <dir with r1m_order.json and locals_XXXX.hdf5> --distractor-desc <r1m_desc_{protocol}.pt>; see REPRODUCE.md for how those are produced and what they cost", |
| "third_party": "the AMES code (github.com/pavelsuma/ames, Apache-2.0), the dinov2_ames.pt checkpoint and the per-dataset dinov2_{query,gallery}_local.hdf5 local descriptors are fetched from the authors on first use and are not redistributed here", |
| "verified": "rerank.py reproduces the no-distractor standard-head two-stage cells on an A10G: ROxford 91.00 / 80.49 and RParis 95.53 / 91.43, matching the values below at the reported precision", |
| "details": "REPRODUCE.md, section 'Two-stage reproduction from this repository'" |
| } |
| }, |
| "training_protocols": { |
| "standard": { |
| "description": "all of GLDv2-clean, the same training data the published systems use", |
| "train_images": 1570970, |
| "train_classes": 76563, |
| "source_artifact": "modal volume fusion-data:/fp_phase3a/full_head_standard.json" |
| }, |
| "decon": { |
| "description": "GLDv2-clean minus the 87 classes our audit matched to ROxford / RParis query landmarks", |
| "train_images": 1568445, |
| "train_classes": 76478, |
| "source_artifact": "modal volume fusion-data:/fp_phase3a/full_head_decon.json" |
| } |
| }, |
| "ours": { |
| "plus_1m": { |
| "gallery": "benchmark database + 1001001 revisitop1m distractors", |
| "source_artifacts": { |
| "standard": "modal volume fusion-data:/fp_phase3a/r1m_eval_standard_dinov2_ames.json", |
| "decon": "modal volume fusion-data:/fp_phase3a/r1m_eval_decon_dinov2_ames.json", |
| "gallery_size": "modal volume fusion-data:/fp_phase3a/r1m/r1m_order.json (total 1001001 over 100 shards)" |
| }, |
| "global_only": { |
| "standard": { |
| "roxford5k": { |
| "M": 67.09, |
| "H": 45.38 |
| }, |
| "rparis6k": { |
| "M": 86.65, |
| "H": 73.88 |
| } |
| }, |
| "decon": { |
| "roxford5k": { |
| "M": 65.62, |
| "H": 41.5 |
| }, |
| "rparis6k": { |
| "M": 87.02, |
| "H": 74.23 |
| } |
| } |
| }, |
| "with_ames_top1600": { |
| "cell": "k1600_l0.55_t0.3", |
| "standard": { |
| "roxford5k": { |
| "M": 87.21, |
| "H": 75.14 |
| }, |
| "rparis6k": { |
| "M": 92.61, |
| "H": 86.03 |
| } |
| }, |
| "decon": { |
| "roxford5k": { |
| "M": 87.11, |
| "H": 73.73 |
| }, |
| "rparis6k": { |
| "M": 92.92, |
| "H": 86.35 |
| } |
| } |
| } |
| }, |
| "no_distractors": { |
| "gallery": "benchmark database only", |
| "source_artifacts": { |
| "global_only": "modal volume fusion-data:/fp_phase3a/full_head_{standard,decon}.json", |
| "with_ames": "modal volume fusion-data:/fp_phase3a/ames_nn_full_{standard,decon}_dinov2_ames.json" |
| }, |
| "global_only": { |
| "standard": { |
| "roxford5k": { |
| "M": 76.51, |
| "H": 58.09 |
| }, |
| "rparis6k": { |
| "M": 92.85, |
| "H": 84.88 |
| } |
| }, |
| "decon": { |
| "roxford5k": { |
| "M": 76.42, |
| "H": 56.18 |
| }, |
| "rparis6k": { |
| "M": 93.22, |
| "H": 85.37 |
| } |
| } |
| }, |
| "with_ames_top1600": { |
| "cell": "k1600_l0.55_t0.3", |
| "standard": { |
| "roxford5k": { |
| "M": 91.0, |
| "H": 80.49 |
| }, |
| "rparis6k": { |
| "M": 95.53, |
| "H": 91.43 |
| } |
| }, |
| "decon": { |
| "roxford5k": { |
| "M": 90.89, |
| "H": 79.55 |
| }, |
| "rparis6k": { |
| "M": 95.53, |
| "H": 91.35 |
| } |
| } |
| } |
| } |
| }, |
| "semantic_search_embedding_comparison": { |
| "question": "whether a multimodal search embedding already covers landmark and place retrieval", |
| "protocol": "same official revisitop harness as every other number in this file: official gnd pickles, 70 bounding-box-cropped queries per dataset, galleries of 4993 and 6322 images, no distractors, global descriptors only, no reranking", |
| "source_artifact": "modal volume fusion-data:/fp_fe2_placerec/results.json (descriptors at fp_fe2_placerec/fe2_{roxford,rparis}.pt)", |
| "script": "scripts/fp_fe2_placerec.py --action {extract,score}", |
| "systems": { |
| "fusion-embedding-2 image tower": { |
| "model": "EximiusLabs/fusion-embedding-2-2b-preview", |
| "revision": "1720d8b16af578d794d7b21ee7b829281941899d", |
| "descriptor": "frozen Qwen3-VL-Embedding-2B image path through the released embed_image API, last-token pooling, MRL prefix truncation, 1024-d", |
| "preprocessing": "the base processor's native image pipeline, aspect ratio preserved, no fixed square crop, no multi-scale averaging; bfloat16 on A10G, scored in float32", |
| "instructions": { |
| "gallery": "Represent the user's input.", |
| "query": "Retrieve images or text relevant to the user's query." |
| }, |
| "read_out": "q_query_cropped|dim1024|centered, the strongest of twelve measured configurations with protocol-cropped queries", |
| "bbox_crop": true, |
| "roxford5k": { |
| "M": 47.52, |
| "H": 18.69 |
| }, |
| "rparis6k": { |
| "M": 78.42, |
| "H": 57.06 |
| } |
| }, |
| "frozen DINOv2 ViT-L/14 CLS, three scales, no training": { |
| "descriptor": "facebook/dinov2-large CLS at scales 1.0 / 1.414 / 2.0, averaged and re-normalized, 1024-d", |
| "bbox_crop": true, |
| "roxford5k": { |
| "M": 72.94, |
| "H": 53.75 |
| }, |
| "rparis6k": { |
| "M": 92.58, |
| "H": 84.02 |
| }, |
| "source_artifact": "modal volume fusion-data:/fp_phase3a/audit_verdict.json, rescore_official" |
| }, |
| "fusion-perception-1 v0.1, standard head": { |
| "descriptor": "the same three-scale CLS input through the trained head, 512-d", |
| "bbox_crop": true, |
| "roxford5k": { |
| "M": 76.51, |
| "H": 58.09 |
| }, |
| "rparis6k": { |
| "M": 92.85, |
| "H": 84.88 |
| }, |
| "source_artifact": "modal volume fusion-data:/fp_phase3a/full_head_standard.json" |
| } |
| }, |
| "read_out_variants": { |
| "note": "twelve configurations per dataset: document instruction on both sides or query instruction on the query side, 1024-d or 2048-d, gallery mean-centering on or off; the uncropped-query rows are a control and are outside the protocol. Full grid in the source artifact.", |
| "weakest_cropped_configuration": { |
| "roxford5k_M": 38.2, |
| "rparis6k_M": 70.18 |
| }, |
| "uncropped_query_control_best_cell": { |
| "roxford5k_H": 22.71 |
| } |
| }, |
| "reading": "The search embedding is behind the head in this repository on all four cells: 28.99 mAP-M and 39.40 mAP-H on ROxford, 14.43 and 27.82 on RParis. It is behind the untrained DINOv2 readout by 25.42 / 35.06 and 14.16 / 26.96, so the difference is not produced by head training. No configuration of the search embedding is competitive on any cell. This measures one task and says nothing about the cross-modal retrieval the search embedding is built for.", |
| "fidelity_check": "the extraction path reproduces the released embed_image API to a maximum absolute difference of 1.5e-8 on sampled images from each dataset, recorded in the source artifact" |
| }, |
| "published_comparison_plus_1m_top1600": { |
| "note": "shortlist size dominates two-stage results, so only rows that rerank a top-1600 shortlist are compared. Values are mAP percent at +1M.", |
| "rows": [ |
| { |
| "system": "RN101-SuperGlobal, global only", |
| "stage": "global only", |
| "roxford5k": { |
| "M": 78.8, |
| "H": 61.9 |
| }, |
| "rparis6k": { |
| "M": 83.9, |
| "H": 69.1 |
| }, |
| "source": "SuperGlobal, ICCV 2023, arXiv 2308.06954, Table 1", |
| "uses_dinov2": false |
| }, |
| { |
| "system": "RN101-SuperGlobal, rerank top-1600", |
| "stage": "two stage", |
| "roxford5k": { |
| "M": 85.9, |
| "H": 74.3 |
| }, |
| "rparis6k": { |
| "M": 87.7, |
| "H": 77.0 |
| }, |
| "source": "SuperGlobal, ICCV 2023, arXiv 2308.06954, supplementary Table 6", |
| "uses_dinov2": false |
| }, |
| { |
| "system": "AMES with CVNet local descriptors, after SuperGlobal rerank, top-1600", |
| "stage": "two stage", |
| "roxford5k": { |
| "M": 86.6, |
| "H": 74.1 |
| }, |
| "rparis6k": { |
| "M": 88.8, |
| "H": 79.9 |
| }, |
| "source": "AMES, ECCV 2024, arXiv 2408.03282, supplementary Table 9", |
| "uses_dinov2": false |
| }, |
| { |
| "system": "AMES with DINOv2 local descriptors, no SuperGlobal rerank, top-1600", |
| "stage": "two stage", |
| "roxford5k": { |
| "M": 87.1, |
| "H": 76.1 |
| }, |
| "rparis6k": { |
| "M": 89.8, |
| "H": 81.0 |
| }, |
| "source": "AMES, ECCV 2024, arXiv 2408.03282, supplementary Table 9", |
| "uses_dinov2": true |
| }, |
| { |
| "system": "AMES with DINOv2 local descriptors, after SuperGlobal rerank, top-1600", |
| "stage": "two stage", |
| "roxford5k": { |
| "M": 88.2, |
| "H": 77.7 |
| }, |
| "rparis6k": { |
| "M": 90.1, |
| "H": 82.0 |
| }, |
| "source": "AMES, ECCV 2024, arXiv 2408.03282, supplementary Table 9", |
| "uses_dinov2": true, |
| "note": "AMES main Table 1 prints only the collapsed ROP+1M average for this configuration; the per-dataset +1M values live in supplementary Table 9. The same values are reproduced in L2G Table 1 as the AMES (600,600) (Re-rank top 1600) row." |
| }, |
| { |
| "system": "L2G CANN-FIRE + MDS rerank", |
| "stage": "two stage", |
| "roxford5k": { |
| "M": 90.5, |
| "H": 79.8 |
| }, |
| "rparis6k": { |
| "M": 92.1, |
| "H": 83.4 |
| }, |
| "source": "L2G, ICCV 2025 Workshop (MRR), arXiv 2509.04351, Table 1", |
| "uses_dinov2": false, |
| "note": "best published +1M numbers we are aware of on both datasets" |
| }, |
| { |
| "system": "fusion-perception-1 v0.1, standard, + AMES top-1600", |
| "stage": "two stage", |
| "roxford5k": { |
| "M": 87.21, |
| "H": 75.14 |
| }, |
| "rparis6k": { |
| "M": 92.61, |
| "H": 86.03 |
| }, |
| "source": "this release", |
| "uses_dinov2": true |
| }, |
| { |
| "system": "fusion-perception-1 v0.1, decon, + AMES top-1600", |
| "stage": "two stage", |
| "roxford5k": { |
| "M": 87.11, |
| "H": 73.73 |
| }, |
| "rparis6k": { |
| "M": 92.92, |
| "H": 86.35 |
| }, |
| "source": "this release", |
| "uses_dinov2": true |
| } |
| ] |
| }, |
| "claims": { |
| "rparis_plus_1m": "Both of our protocols exceed the best published RParis+1M numbers on Medium and on Hard. Against L2G (92.1 / 83.4): standard +0.51 M and +2.63 H, decon +0.82 M and +2.95 H.", |
| "roxford_plus_1m": "ROxford+1M Medium is level with AMES using DINOv2 locals without SuperGlobal reranking (87.21 and 87.11 against 87.1) and 3.3 to 3.4 points below L2G. ROxford+1M Hard is 0.96 to 2.37 points below that AMES row and 4.7 to 6.1 points below L2G.", |
| "reranking_contribution": "AMES reranking of a top-1600 shortlist adds 20.12 M on ROxford+1M and 5.96 M on RParis+1M over our own global-only ranking (standard protocol).", |
| "not_claimed": "This is not a general state-of-the-art claim. It is specific to RParis+1M, and it rests on a DINOv2 backbone whose pretraining data includes ROxford and RParis imagery (see contamination below)." |
| }, |
| "contamination": { |
| "gldv2_train_test_overlap": { |
| "what_we_measured": "87 GLDv2-clean classes containing 2529 images match ROxford or RParis query landmarks", |
| "source_artifact": "modal volume fusion-data:/fp_phase3a/plan/plan_full.json (leak_classes 87, leak_images_total 2529, clean_classes 81313, clean_images 1580470); class list at fp_phase3a/plan/leak_class_ids.json", |
| "published_baseline": "RGLDv2-clean removes 1565 images from 18 GIDs of GLDv2-clean (Song et al., On Train-Test Class Overlap and Detection for Image Retrieval, CVPR 2024, arXiv 2404.01524, page 4)", |
| "what_we_do": "the decon head excludes those 87 classes from training; the standard head does not, so it is directly comparable to published systems", |
| "published_class_list": { |
| "file": "leak_class_ids.json", |
| "contents": "all 87 excluded classes with GLDv2 landmark id, Wikimedia Commons category and the matching rule, so the exclusion set is checkable class by class", |
| "matching_stance": "inclusive: a class is excluded when it plausibly depicts a queried landmark, without image-level duplicate detection", |
| "source_artifacts": [ |
| "fusion-data:/fp_phase2/plan/leakage_report.json", |
| "fusion-data:/fp_phase3a/plan/leak_class_ids.json", |
| "fusion-data:/fp_phase3a/plan/plan_full.json" |
| ] |
| }, |
| "image_level_verification": { |
| "image_precision": 0.43, |
| "image_precision_ci95": [ |
| 0.399, |
| 0.452 |
| ], |
| "class_precision": "13 of 87 (23 counting structures in the same complex)", |
| "by_rule_image_precision": { |
| "unique:*": 0.58, |
| "blanket:oxford": 0.18, |
| "ctx:*": 0.08 |
| }, |
| "visual_audit": "170 stratified random flagged images judged against the query crops: 0.43 +/- 0.08, Cohen's kappa 0.84 with the automatic detector", |
| "rgldv2_containment": "contains 10 of RGLDv2-clean's 18 landmark ids and 1,216 of its 1,565 images; NOT a superset", |
| "matcher_recall": "757 of 995 contaminated images found by the calibrated detector across all of GLDv2-clean, 76%", |
| "interpretation": "the decon head is a conservative ablation bounding the cost of removing overlapping training data from above, not a stricter cleanup than RGLDv2-clean", |
| "source_artifact": "fusion-data:/fp_phase3a/leak_verify/leak_verify_results.json" |
| } |
| }, |
| "backbone_pretraining": { |
| "what": "DINOv2's LVD-142M pretraining set contains GLDv2-clean as is (1580470 images) plus 6321880 sampled from it, and 1000000 images retrieved with Revisiting Oxford base (4993 images) and 1000000 retrieved with Revisiting Paris base (6322 images) as seeds", |
| "source": "DINOv2, arXiv 2304.07193, Table 15", |
| "ames_position": "AMES declined DINOv2 as its default global descriptor for this reason: \"We do not consider DINOv2 as the default option since ROxford and RParis are listed among the datasets used for its self-supervised training\" (arXiv 2408.03282, section 4)", |
| "asymmetry": "our numbers use DINOv2 for both the global descriptor and the local descriptors, so the caveat applies fully. It also applies to the AMES rows that use DINOv2 locals, including their best +1M row. It does not apply to L2G, which uses FIRE local features and no DINOv2, nor to SuperGlobal, nor to the AMES row with CVNet locals." |
| }, |
| "seed_spread": { |
| "design": "3 seeds per protocol, head training only, seed the sole variable", |
| "no_distractor_global_only_mean_sd": { |
| "standard": { |
| "roxford": { |
| "M": [ |
| 76.07, |
| 0.9 |
| ], |
| "H": [ |
| 56.68, |
| 1.16 |
| ] |
| }, |
| "rparis": { |
| "M": [ |
| 93.2, |
| 0.25 |
| ], |
| "H": [ |
| 85.71, |
| 0.51 |
| ] |
| } |
| }, |
| "decon": { |
| "roxford": { |
| "M": [ |
| 76.41, |
| 0.2 |
| ], |
| "H": [ |
| 56.08, |
| 0.73 |
| ] |
| }, |
| "rparis": { |
| "M": [ |
| 93.28, |
| 0.03 |
| ], |
| "H": [ |
| 86.12, |
| 0.18 |
| ] |
| } |
| } |
| }, |
| "verdict": "3 of 4 standard-vs-decon deltas fall below the within-condition seed spread and the fourth is 1.1 pooled sd; supports a bound (no cell moves more than ~1 mAP) rather than a null result", |
| "shipped_heads": "one run each, trained without a fixed seed. On RParis the shipped standard head is the weakest of four runs; on ROxford Hard it is the strongest.", |
| "selection_artifact": "ROxford M and RParis M correlate at r = -0.74 across the six runs because the epoch is selected on ROxford", |
| "source_artifact": "fusion-data:/fp_phase3a/seeds/seed_analysis.json" |
| } |
| }, |
| "provenance": { |
| "backbone": "facebook/dinov2-large, frozen, Apache-2.0", |
| "trained_weights": "one MLP head, 3148800 parameters, shipped in two training protocols", |
| "reranker": "AMES public checkpoint dinov2_ames.pt and the authors' published DINOv2-B local descriptors, used unchanged; not redistributed here", |
| "code_verification": "inference.py in this repo reproduces the exact descriptors used for the reported evaluation: cosine 0.999995 to 1.000006 against the cached ROxford gallery descriptors on three sampled images, and 0.9996 to 0.99996 on two sampled bounding-box-cropped queries" |
| } |
| } |
|
|
