Add files using upload-large-folder tool

.env.example

@@ -0,0 +1,31 @@
+# Example variable names only. Do not put real credentials in this file.
+# In this workspace, use openrouter-run for OpenRouter operations.
+
+# Main/legacy provider paths
+# OPENROUTER_API_KEY=<inject through a secret manager or process environment>
+OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
+OPENROUTER_MODEL=openai/gpt-5.6-sol
+ORIENT_MODEL_PROFILE=default
+# OPENAI_API_KEY=
+# GOOGLE_API_KEY=
+# ANTHROPIC_API_KEY=
+# ZHIPU_API_KEY=
+ORIENTER_APP_METADATA_CACHE=
+
+# Semantic evaluation cache. Keep offline/read-only for historical reproduction.
+# Set ZHIPU_API_KEY only when intentionally expanding the cache for new labels.
+ORIENTER_EMBEDDING_CACHE=
+ORIENTER_EMBEDDING_OFFLINE=1
+ORIENTER_EMBEDDING_READONLY=1
+
+# Optional baseline/provider paths
+# ARK_API_KEY=
+# DASHSCOPE_API_KEY=
+# DEEPSEEK_API_KEY=
+# THEB_API_KEY=
+# INTERNVL_API_KEY=
+CLAUDE_BASELINE_MODEL=anthropic/claude-sonnet-4.5
+GPT4O_BASELINE_MODEL=openai/gpt-4o-2024-08-06
+GPT5_BASELINE_MODEL=openai/gpt-5.6-sol
+GEMINI_BASELINE_MODEL=google/gemini-2.5-pro
+D3_DATASET_ROOT=

.gitattributes

@@ -1,35 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+approach/ovod/APE/ape_d_model_final.pth filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,69 @@
+# Credentials and local configuration
+.env
+.env.*
+!.env.example
+AGENTS.md
+*.pem
+*.key
+
+# Python/editor state
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.mypy_cache/
+.ipynb_checkpoints/
+.vscode/
+.idea/
+.DS_Store
+.nfs*
+
+# Virtual environments and package/build output
+.venv/
+venv/
+env/
+build/
+dist/
+*.egg-info/
+
+# Logs, caches, and scratch output
+*.log
+*.tmp
+*.bak
+.cache/
+log/
+logs/
+fastimg/
+wandb/
+work_dirs/
+**/output/
+**/outputs/
+**/runs/
+
+# Model and archive payloads
+*.pt
+*.pth
+!approach/ovod/APE/ape_d_model_final.pth
+*.ckpt
+*.onnx
+*.bin
+*.pkl
+*.zip
+*.7z
+*.tar
+*.tar.gz
+
+# Data payloads (keep scripts and small split metadata in Git)
+/data/
+dataset/data/
+
+# Generated model/API and evaluation artifacts
+approach/vlm/*.jsonl
+approach/llm/*.json
+approach/ovod/*.json
+evaluation/embedding_dict*.json
+evaluation/results/
+evaluation/eval_results/
+evaluation/autotest_study/results*/
+evaluation/eval_context/results/
+evaluation/eval_latex/**/*.csv
+evaluation/eval_latex/**/*.tex

CITATION.cff

@@ -0,0 +1,69 @@
+cff-version: 1.2.0
+message: "If you use Orienter, please cite the accompanying paper."
+title: "Orienter: Context-Sensitive GUI Grounding for Automated XR Testing"
+type: software
+authors:
+  - family-names: Li
+    given-names: Shuqing
+    orcid: "https://orcid.org/0000-0001-6323-1402"
+  - family-names: Li
+    given-names: Binchang
+    orcid: "https://orcid.org/0009-0008-5995-4040"
+  - family-names: Liu
+    given-names: Yepang
+    orcid: "https://orcid.org/0000-0001-8147-8126"
+  - family-names: Gao
+    given-names: Cuiyun
+    orcid: "https://orcid.org/0000-0003-4774-2434"
+  - family-names: Zhang
+    given-names: Jianping
+    orcid: "https://orcid.org/0009-0005-5978-8888"
+  - family-names: Cheung
+    given-names: Shing-Chi
+    orcid: "https://orcid.org/0000-0002-3508-7172"
+  - family-names: Lyu
+    given-names: Michael R.
+    orcid: "https://orcid.org/0000-0002-3666-5798"
+abstract: >-
+  Orienter grounds context-sensitive, user-interactable GUI elements in
+  extended-reality screenshots for automated XR testing.
+keywords:
+  - extended reality
+  - GUI grounding
+  - software testing
+  - vision-language models
+preferred-citation:
+  type: article
+  title: "Look Before You Leap: Context-Sensitive GUI Grounding for Boosting Automated Extended Reality (XR) Testing"
+  authors:
+    - family-names: Li
+      given-names: Shuqing
+      orcid: "https://orcid.org/0000-0001-6323-1402"
+    - family-names: Li
+      given-names: Binchang
+      orcid: "https://orcid.org/0009-0008-5995-4040"
+    - family-names: Liu
+      given-names: Yepang
+      orcid: "https://orcid.org/0000-0001-8147-8126"
+    - family-names: Gao
+      given-names: Cuiyun
+      orcid: "https://orcid.org/0000-0003-4774-2434"
+    - family-names: Zhang
+      given-names: Jianping
+      orcid: "https://orcid.org/0009-0005-5978-8888"
+    - family-names: Cheung
+      given-names: Shing-Chi
+      orcid: "https://orcid.org/0000-0002-3508-7172"
+    - family-names: Lyu
+      given-names: Michael R.
+      orcid: "https://orcid.org/0000-0002-3666-5798"
+  journal: "Proceedings of the ACM on Software Engineering"
+  volume: 3
+  issue: FSE
+  number: FSE127
+  start: 2858
+  end: 2881
+  month: 6
+  year: 2026
+  doi: "10.1145/3808134"
+  url: "https://doi.org/10.1145/3808134"

PIPELINE_README.md

@@ -0,0 +1,224 @@
+# Orienter pipeline guide
+
+This guide maps the public commands to the method described in the paper. It distinguishes the configurable release path from historical experiment scripts and precomputed artifacts.
+
+## Data contracts
+
+The pipeline uses three small JSON formats.
+
+Question manifest (`questions.jsonl`):
+
+```json
+{"question_id": 0, "image": "123_4.jpg", "image_id": 123004, "text": "Identify interactable elements."}
+```
+
+Candidate output (`candidates.jsonl`):
+
+```json
+{"question_id": 0, "text": {"objects": {"red button": "round red button with white text"}}, "model_id": "openai/gpt-5.6-sol"}
+```
+
+Detection output (`predictions.json`):
+
+```json
+[{"image_id": 123004, "category_id": "red button", "category_name": "red button", "bbox": [10, 20, 30, 40], "score": 0.91}]
+```
+
+Bounding boxes use COCO `x, y, width, height`. `image_id` is derived from `<app_id>_<frame>.ext` by zero-padding a frame in `[0, 999]` to three digits. Filenames must contain exactly one underscore between numeric app ID and numeric frame ID.
+
+Keep all mounted inputs and generated artifacts outside the repository, then create the question manifest:
+
+```bash
+export ORIENTER_DATA_DIR=/absolute/path/to/orienter_data
+export ORIENTER_RUN_DIR=/absolute/path/to/orienter_run
+mkdir -p "$ORIENTER_RUN_DIR"
+
+python -B scripts/generate_questions.py \
+  --images-dir "$ORIENTER_DATA_DIR/images" \
+  --output "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --prompt "Identify interactable elements." \
+  --metadata-template "$ORIENTER_RUN_DIR/app_metadata.json"
+```
+
+Screenshot filenames must match `<numeric_app_id>_<numeric_frame>.<supported_ext>` and use a supported image extension. The manifest is sorted by derived `image_id` so repeated generation is deterministic. Fill the generated metadata template with a non-empty app name and description for every app ID before Stage 1.
+
+## Stage 0: external assets
+
+Before a live run, provide:
+
+- XR screenshots and a matching question manifest;
+- application metadata reachable from Steam, or a pinned JSON/JSONL metadata cache;
+- the APE-L\(_D\) repository environment (the checkpoint itself is bundled);
+- an OpenRouter credential for the selected LMM profile;
+- the bundled ground-truth split matching the experiment, plus a reviewed semantic-embedding cache. Credentials are needed only when intentionally adding embeddings for previously unseen labels.
+
+The repository includes the main checkpoint and selected evaluation ground truth. It intentionally excludes the screenshot payload, private context data, generated predictions, experiment results, and the semantic embedding cache. Exact requirements are listed in `docs/ASSETS.md`.
+
+## Stage 1: context-sensitive candidate mining
+
+Run:
+
+```bash
+openrouter-run python -B -m approach.run_vlm \
+  --questions "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --images-dir "$ORIENTER_DATA_DIR/images" \
+  --output "$ORIENTER_RUN_DIR/candidates.jsonl" \
+  --profile default \
+  --app-metadata-cache "$ORIENTER_RUN_DIR/app_metadata.json"
+```
+
+`approach/run_vlm.py` selects the configured model profile, loads each screenshot, combines screenshot and application context with the research prompt, and atomically checkpoints JSONL responses. It accepts:
+
+- `--start-index/--end-index` for a contiguous range;
+- `--shard-index/--num-shards` for deterministic modulo sharding;
+- `--resume` for completed `question_id` values;
+- `--app-metadata-cache` for a pinned JSON/JSONL app metadata cache;
+- `--ablation` for the historical no-interactability prompt.
+
+Range and shard modes cannot be combined. Every selected output path receives a range/shard suffix.
+
+The default profile uses OpenRouter. The optional paper profiles preserve the evaluated identifiers for GPT-4o-2024-08-06, Claude 3.5 Sonnet, and Gemini 1.5 Pro. Because provider catalogs can retire historical routes, archive the resolved provider/model metadata with every reproduction; if a paper route is unavailable, supply a custom YAML/JSON profile or direct vendor adapter and document the substitution.
+
+For reproducible runs, provide `--app-metadata-cache` or set `ORIENTER_APP_METADATA_CACHE`. The cache may be a JSON object keyed by app ID or JSONL records with `app_id`, `app_name`/`name`/`title`, and `app_description`/`description`. When this option is set, duplicate or missing app IDs fail before the provider call and the runner does not fall back to live Steam pages.
+
+## Stage 2: APE-L_D visual grounding
+
+Run:
+
+```bash
+python -B -m approach.run_ape \
+  --questions "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --candidates "$ORIENTER_RUN_DIR/candidates.jsonl" \
+  --images-dir "$ORIENTER_DATA_DIR/images" \
+  --output "$ORIENTER_RUN_DIR/predictions.json" \
+  --visualization-dir "$ORIENTER_RUN_DIR/ape_visualizations" \
+  --ape-root approach/ovod/APE \
+  --ape-config configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py \
+  --ape-checkpoint ape_d_model_final.pth \
+  --threshold 0.15
+```
+
+The runner:
+
+1. accepts candidate `text` as a JSON object or JSON string;
+2. converts object descriptions to APE referring expressions;
+3. invokes APE once per selected screenshot;
+4. enriches every result with `image_id` and `category_id`;
+5. writes one atomic JSON document plus a progress sidecar;
+6. records per-image failures separately without corrupting successful predictions.
+
+This fixes the historical failure mode where `all_ape_res` was cleared inside the loop, multiple JSON documents were dumped into one file, and the raw `ape_item` was appended instead of the enriched item.
+
+## PII.5–PII.7 reflection
+
+Enable reflection with:
+
+```bash
+openrouter-run python -B -m approach.run_ape ... \
+  --enable-reflection \
+  --reflection-profile default \
+  --max-reflection-iterations 10
+```
+
+The implementation follows the paper:
+
+- **PII.5, mirroring-based regional verification:** crop each detected element inside its bbox and send the original scene plus crop as two separate images. The crop keeps its original pixel dimensions and orientation; it is not enlarged, resized, or flipped.
+- **PII.6, unsuccessful-detection reflection:** send the full scene with current boxes so the advisor can identify missed elements and incorrect regions.
+- **PII.7, advisor feedback:** require structured `verified`, `needs_refinement`, and `feedback` fields. False positives listed in `needs_refinement` are removed; verified detections are retained; feedback descriptions are grounded again by APE.
+
+The loop stops when no refinement or feedback remains, or after at most 10 advisor rounds by default. Reflection is opt-in because it incurs additional API cost. If the advisor call fails in the configurable runner, the original APE detections are preserved and the reflection error is recorded.
+
+## Parallel shards and validated merge
+
+Example four-way detection:
+
+```bash
+for shard in 0 1 2 3; do
+  python -B -m approach.run_ape ... --num-shards 4 --shard-index "$shard"
+done
+```
+
+Each output is named `.shardNN-of-NN.json`; each progress sidecar also includes screenshots with zero detections. Merge with:
+
+```bash
+python -B -m evaluation.tools.merge_predictions \
+  --inputs "$ORIENTER_RUN_DIR/predictions.shard00-of-04.json" \
+           "$ORIENTER_RUN_DIR/predictions.shard01-of-04.json" \
+           "$ORIENTER_RUN_DIR/predictions.shard02-of-04.json" \
+           "$ORIENTER_RUN_DIR/predictions.shard03-of-04.json" \
+  --questions "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --output "$ORIENTER_RUN_DIR/predictions.merged.json"
+```
+
+The merge fails closed on overlapping completed images, duplicate detections, missing progress files, and missing expected images. `--allow-incomplete` exists for deliberate partial diagnostics.
+
+## Stage 3: task conversion and evaluation
+
+Convert the merged list explicitly:
+
+```bash
+python -B -m evaluation.tools.to_pred \
+  --task all \
+  --input "$ORIENTER_RUN_DIR/predictions.merged.json" \
+  --output "$ORIENTER_RUN_DIR/predictions.json"
+```
+
+This produces separate `semantics`, `interactable`, and `interaction` files. The task can no longer be changed accidentally by two consecutive assignments in source code.
+
+The remaining evaluation sequence is:
+
+1. `evaluation/filter_box.py` applies NMS and geometric filtering;
+2. `evaluation/filter_test_prediction.py` selects the requested split;
+3. `evaluation/evaluate_coco.py` computes modified COCO metrics;
+4. `evaluation/context_eval.py` performs context-sensitive semantic matching;
+5. `evaluation/autotest_study/` performs the RQ3 interaction simulation.
+
+The historical shell scripts retain paper experiment names, but their data stems depend on external prediction/GT assets. Prefer explicit Python tool arguments for new runs.
+
+### Offline semantic evaluation
+
+Semantic evaluation can run without an API call by mounting the reviewed historical cache described in `evaluation/cache_manifest.json`:
+
+```bash
+export ORIENTER_EMBEDDING_CACHE=/absolute/path/to/embedding_dict.json
+export ORIENTER_EMBEDDING_OFFLINE=1
+export ORIENTER_EMBEDDING_READONLY=1
+unset ZHIPU_API_KEY
+
+python -B evaluation/evaluate_coco.py \
+  -gt /absolute/path/to/semantics_test.json \
+  -dt /absolute/path/to/predictions.json \
+  -d s -s -l "$ORIENTER_RUN_DIR/semantic_eval.csv"
+```
+
+Offline mode fails before any provider call when a category is absent from the cache. Keep the reviewed cache read-only. If a genuinely new model introduces an uncached label, expand a separate working copy deliberately with `ZHIPU_API_KEY`, review it, record a new checksum, and then freeze that new version.
+
+Verify the mounted historical cache against the frozen manifest before evaluation:
+
+```bash
+python -B scripts/verify_assets.py \
+  --repo-root . \
+  --skip-checkpoint \
+  --embedding-cache /absolute/path/to/embedding_dict.json
+```
+
+The legacy `make_sample_gt.py` path keeps only image IDs present in a method's prediction file. It is retained for historical protocol compatibility, but evaluates different methods on different GT subsets. New comparisons must score every method against the same complete GT and report prediction coverage. Historical predictions, generated CSVs, and validation summaries are not bundled.
+
+## Legacy experiment provenance
+
+`approach/method*.py` contains commented branches for earlier datasets, ablations, machines, and model variants. Those comments were retained because a disabled block may represent another pipeline stage rather than dead code.
+
+For release use:
+
+- `run_vlm.py` replaces source-comment switching for screenshot-context generation;
+- `run_ape.py` replaces source-comment switching for detector paths, ranges, shards, and reflection;
+- explicit model profiles replace provider/model edits;
+- the merge and conversion tools replace hand-edited evaluation state.
+
+The legacy entrypoints remain available for provenance and now contain the APE accumulation fix and CLI range/shard controls, but they are not the recommended fresh-run interface.
+
+## Verification boundary
+
+Offline tests cover provider payloads, model aliases, PII.5 crop fidelity, false-positive removal, loop termination, APE accumulation/resume, shard isolation/merge, evaluation conversion, fail-closed semantic-cache access, and process-isolated evaluator temporary files.
+
+The bundled tests verify evaluation mechanics without retaining prior prediction or metric artifacts. Paper-scale validation still requires fresh model inference, fixed-GT comparisons, and separately archived provenance. Selected evaluation ground truth is bundled, but the screenshot payload and authoritative public asset manifest are external.

README.md

@@ -0,0 +1,245 @@
+---
+tags:
+  - computer-vision
+  - object-detection
+  - multimodal
+  - extended-reality
+  - graphical-user-interface
+---
+
+# Orienter
+
+Official code release for **“Look Before You Leap: Context-Sensitive GUI Grounding for Boosting Automated Extended Reality (XR) Testing.”**
+
+[Paper](https://doi.org/10.1145/3808134) · [Project page](https://sites.google.com/view/gui-orienter) · [Pipeline guide](PIPELINE_README.md) · [Environment](docs/ENVIRONMENT.md) · [Required assets](docs/ASSETS.md) · [Model manifest](docs/MODEL_MANIFEST.md) · [Release checklist](docs/RELEASE_CHECKLIST.md)
+
+Orienter detects context-sensitive, user-interactable GUI elements in XR screenshots. The public pipeline has three explicit stages:
+
+1. an LMM combines application context and the current screenshot to mine interactable element descriptions;
+2. APE-L\(_D\) grounds those descriptions, with the optional PII.5–PII.7 reflection loop described in the paper;
+3. the evaluation tools convert, merge, filter, and score predictions.
+
+The repository contains source code, selected evaluation ground truth, and the APE-L\(_D\) checkpoint needed by the main detector. Dataset image payloads, private data, API credentials, experiment logs, generated predictions, and previous experimental results are deliberately excluded.
+
+## Release status
+
+The offline control path and utilities are unit-tested. Maintainer validation has exercised one-image live paths for the default OpenRouter VLM, APE-L\(_D\), and the PII.5–PII.7 advisor loop without retaining generated artifacts in this public copy. Historical experiment outputs and validation result tables are not bundled.
+
+A fresh-clone paper-scale inference reproduction still requires the external assets in [docs/ASSETS.md](docs/ASSETS.md), especially the dataset image payload, provider access, and a validated CUDA/APE environment.
+
+The paper is published in PACMSE, Volume 3, Issue FSE, Article FSE127 ([DOI 10.1145/3808134](https://doi.org/10.1145/3808134)). Do not publish this directory until the repository license, third-party revision audit, and final public model/data links are complete. Machine-readable citation metadata is available in [`CITATION.cff`](CITATION.cff).
+
+## Repository layout
+
+| Path | Purpose |
+| --- | --- |
+| `approach/run_vlm.py` | configurable screenshot-context and candidate generation |
+| `approach/run_ape.py` | configurable APE detection and optional reflection |
+| `approach/reflection.py` | paper-aligned PII.5, PII.6, and PII.7 loop |
+| `approach/config.py` | OpenRouter and paper model profiles |
+| `approach/method*.py` | legacy experiment entrypoints retained for provenance |
+| `approach/ovod/` | vendored APE, Grounding DINO, and related detector code |
+| `evaluation/tools/` | prediction conversion and validated shard merge |
+| `evaluation/` | detection, semantic, context, and interaction evaluation |
+| `dataset/` | dataset conversion and split-generation scripts |
+| `tests/` | offline regression tests for release-critical logic |
+
+## Installation
+
+The supported public environment targets Python 3.10 or later so it can use a currently supported Pillow release with the latest security fixes:
+
+```bash
+python -B -m venv .venv
+source .venv/bin/activate
+python -B -m pip install --upgrade pip
+python -B -m pip install -r requirements.txt
+```
+
+`requirements.txt` covers the release utilities and provider adapters. APE-L\(_D\) has a separate CUDA/PyTorch stack under `approach/ovod/APE`; the checkpoint is bundled at `approach/ovod/APE/ape_d_model_final.pth` and recorded in [docs/MODEL_MANIFEST.md](docs/MODEL_MANIFEST.md). The public tree intentionally excludes compiled APE extensions, build directories, and egg-info metadata, so build/install the APE extension inside the target CUDA environment before running Stage 2. See [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) and `environment.reference.yml` for the tested historical environment, a public compatibility starting point, build commands, and verification boundary. The reference YAML is not a cross-platform lockfile or a security attestation. The historical combined dependency snapshot remains at `approach/requirements.txt`, but it is not a clean lockfile.
+
+## Model configuration
+
+The default provider is OpenRouter. In this project workspace, use `openrouter-run` so the key is injected by the local credential wrapper and never written to source files, logs, or shell history:
+
+```bash
+openrouter-run python -B -m approach.run_vlm --help
+```
+
+For a public fresh clone outside this workspace, inject `OPENROUTER_API_KEY` through the runner's secret manager or process environment. Do not commit `.env`, shell history, logs, or generated manifests containing credential values.
+
+The default profile uses `openai/gpt-5.6-sol`, the current flagship model configured for this release. Override it without editing code:
+
+```bash
+export OPENROUTER_MODEL=openai/gpt-5.6-sol
+```
+
+Paper-oriented profiles remain available through OpenRouter:
+
+| Profile | Model |
+| --- | --- |
+| `paper_openai` / `gpt4v` | `openai/gpt-4o-2024-08-06` |
+| `paper_claude` / `claude35sonnet` | `anthropic/claude-3.5-sonnet` |
+| `paper_gemini` / `gemini15pro` | `google/gemini-pro-1.5` |
+
+Profiles are defined in `approach/config.py`. Library callers can override the catalog with a YAML or JSON profile file; CLI users select a named profile and may override the default model through `OPENROUTER_MODEL`. The paper profiles preserve the evaluated identifiers for GPT-4o-2024-08-06, Claude 3.5 Sonnet, and Gemini 1.5 Pro. Provider catalogs may retire historical routes; if one is unavailable, use a custom profile or direct vendor access and record the substitution instead of silently changing the paper profile. The client records only the credential variable name in run manifests; it never writes the credential value.
+
+## Quick start
+
+The commands below show the complete public interface. They require the external files listed in [docs/ASSETS.md](docs/ASSETS.md). Keep screenshots, metadata, prompts, predictions, visualizations, and evaluation files outside the repository:
+
+```bash
+export ORIENTER_DATA_DIR=/absolute/path/to/orienter_data
+export ORIENTER_RUN_DIR=/absolute/path/to/orienter_run
+mkdir -p "$ORIENTER_RUN_DIR"
+```
+
+Build a question manifest from mounted screenshots:
+
+```bash
+python -B scripts/generate_questions.py \
+  --images-dir "$ORIENTER_DATA_DIR/images" \
+  --output "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --prompt "Identify interactable elements." \
+  --metadata-template "$ORIENTER_RUN_DIR/app_metadata.json"
+```
+
+Screenshot filenames must match `<numeric_app_id>_<numeric_frame>.<supported_ext>`, for example `123_4.jpg`; additional underscores are not supported. The frame must be in `[0, 999]` and is zero-padded to three digits when deriving `image_id`. Fill every generated `app_name` and `app_description` field in `$ORIENTER_RUN_DIR/app_metadata.json` before Stage 1; blank or missing records fail closed.
+
+Validate the mounted inputs and the metadata coverage:
+
+```bash
+python -B scripts/verify_assets.py \
+  --repo-root . \
+  --questions "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --images-dir "$ORIENTER_DATA_DIR/images" \
+  --app-metadata-cache "$ORIENTER_RUN_DIR/app_metadata.json"
+```
+
+Generate candidate descriptions:
+
+```bash
+openrouter-run python -B -m approach.run_vlm \
+  --questions "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --images-dir "$ORIENTER_DATA_DIR/images" \
+  --output "$ORIENTER_RUN_DIR/candidates.jsonl" \
+  --profile default \
+  --app-metadata-cache "$ORIENTER_RUN_DIR/app_metadata.json"
+```
+
+`--app-metadata-cache` accepts JSON or JSONL records keyed by `app_id` and avoids live Steam page drift. When supplied, missing app IDs fail closed. Omit it only for exploratory runs where live Steam lookup is acceptable.
+
+### Data privacy
+
+Stage 1 sends each source screenshot to the selected OpenRouter/provider model. When reflection is enabled, PII.5 additionally sends the bounding-box crop and PII.6 sends the annotated full scene. Do not run private, personally identifying, confidential, or otherwise unauthorized screenshots through a remote provider. Review the selected provider's retention and data-processing terms, obtain the necessary permission, and use an approved local/provider deployment when screenshots cannot leave your environment. Generated candidates, predictions, traces, and error files may also reveal image labels or local filenames; keep them outside the release repository.
+
+Ground candidates with APE-L\(_D\):
+
+```bash
+python -B -m approach.run_ape \
+  --questions "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --candidates "$ORIENTER_RUN_DIR/candidates.jsonl" \
+  --images-dir "$ORIENTER_DATA_DIR/images" \
+  --output "$ORIENTER_RUN_DIR/predictions.json" \
+  --visualization-dir "$ORIENTER_RUN_DIR/ape_visualizations" \
+  --ape-root approach/ovod/APE \
+  --ape-checkpoint ape_d_model_final.pth
+```
+
+The released checkpoint is SHA-256 verified before loading. A different checkpoint is rejected unless `--trust-custom-checkpoint` is supplied explicitly after independently verifying its source; PyTorch checkpoints are executable pickle-based artifacts and must not be loaded from an untrusted source.
+
+Enable the paper reflection loop explicitly because it incurs additional LMM calls:
+
+```bash
+openrouter-run python -B -m approach.run_ape \
+  --questions "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --candidates "$ORIENTER_RUN_DIR/candidates.jsonl" \
+  --images-dir "$ORIENTER_DATA_DIR/images" \
+  --output "$ORIENTER_RUN_DIR/predictions.json" \
+  --visualization-dir "$ORIENTER_RUN_DIR/ape_visualizations" \
+  --ape-checkpoint ape_d_model_final.pth \
+  --enable-reflection \
+  --reflection-profile default \
+  --max-reflection-iterations 10
+```
+
+PII.5 sends the original scene and the bbox crop as separate images. The crop is neither enlarged nor flipped. PII.6 sends the full scene with current boxes, and PII.7 returns structured verification/refinement feedback. Rejected false positives are removed before the next detector call; verified detections are retained.
+
+## Ranges, shards, resume, and merge
+
+Use either a range or deterministic modulo shards; the two selection modes are intentionally mutually exclusive. Output filenames receive a selection suffix automatically, so parallel jobs cannot overwrite each other.
+
+```bash
+python -B -m approach.run_ape ... --start-index 0 --end-index 100
+python -B -m approach.run_ape ... --num-shards 4 --shard-index 0
+python -B -m approach.run_ape ... --num-shards 4 --shard-index 1
+```
+
+Add `--resume` to continue from an existing output. Progress sidecars record completed images even when a valid inference produces zero detections.
+
+Merge shards only after validating overlap and coverage:
+
+```bash
+python -B -m evaluation.tools.merge_predictions \
+  --inputs "$ORIENTER_RUN_DIR/predictions.shard00-of-04.json" \
+           "$ORIENTER_RUN_DIR/predictions.shard01-of-04.json" \
+           "$ORIENTER_RUN_DIR/predictions.shard02-of-04.json" \
+           "$ORIENTER_RUN_DIR/predictions.shard03-of-04.json" \
+  --questions "$ORIENTER_RUN_DIR/questions.jsonl" \
+  --output "$ORIENTER_RUN_DIR/predictions.merged.json"
+```
+
+The merge command fails on overlapping shard progress, duplicate detections, missing sidecars, or incomplete expected image coverage.
+
+## Evaluation
+
+Convert prediction fields explicitly for each task:
+
+```bash
+python -B -m evaluation.tools.to_pred \
+  --task semantics \
+  --input "$ORIENTER_RUN_DIR/predictions.merged.json" \
+  --output "$ORIENTER_RUN_DIR/predictions.semantics.json"
+```
+
+`--task` accepts `semantics`, `interactable`, `interaction`, or `all`; `all` writes separate outputs and never overwrites one task with another. The remaining metric scripts and expected ground-truth layouts are documented in [PIPELINE_README.md](PIPELINE_README.md).
+
+## Tests
+
+Run the release-critical offline suite from the repository root:
+
+```bash
+python -B scripts/smoke_control_path.py
+python scripts/run_tests.py
+```
+
+The deterministic control-path smoke uses a generated image plus injected provider/detector functions; it does not call a paid API or load APE weights. The unit tests are also offline. The final release gate additionally requires a real one-image inference, reflection smoke test, and evaluation smoke test after the external materials are supplied.
+
+## Legacy experiment scripts
+
+`approach/method.py`, `method_fastuse.py`, `method_claude.py`, and `method_gemini.py` preserve historical branches and commented stage variants so experiment provenance is not lost. Their active APE paths now produce valid accumulated JSON and accept range/shard controls; `method.py` and `method_fastuse.py` also accept explicit data/checkpoint/output paths and reflection flags.
+
+For new runs, prefer `run_vlm.py` and `run_ape.py`. They make stage selection, paths, model profiles, app metadata caches, ranges, and shards configurable instead of requiring source edits.
+
+## Data, licenses, and citation
+
+Dataset images, annotations, bundled weights, and vendored third-party code may use different licenses. Publish every external asset with its stable URL, version, SHA-256, license, and expected local path. See [docs/THIRD_PARTY.md](docs/THIRD_PARTY.md) and [docs/MODEL_MANIFEST.md](docs/MODEL_MANIFEST.md).
+
+The paper is published in *Proceedings of the ACM on Software Engineering*, Volume 3, Issue FSE, Article FSE127, pages 2858–2881, in 2026:
+
+```bibtex
+@article{li2026look,
+  author  = {Shuqing Li and Binchang Li and Yepang Liu and Cuiyun Gao and
+             Jianping Zhang and Shing-Chi Cheung and Michael R. Lyu},
+  title   = {Look Before You Leap: Context-Sensitive GUI Grounding for
+             Boosting Automated Extended Reality (XR) Testing},
+  journal = {Proceedings of the ACM on Software Engineering},
+  year    = {2026},
+  volume  = {3},
+  number  = {FSE},
+  pages   = {2858--2881},
+  articleno = {FSE127},
+  doi     = {10.1145/3808134}
+}
+```
+
+`CITATION.cff` contains the same preferred paper citation. The top-level code license and final Hugging Face repository links remain publication decisions; licenses for the paper, data, model, and vendored source are separate and must not be inferred from one another.

approach/app_metadata.py

@@ -0,0 +1,95 @@
+"""Load and validate pinned application metadata used by Orienter prompts."""
+
+import json
+from collections.abc import Mapping
+from pathlib import Path
+
+
+def _reject_duplicate_keys(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError(f"Duplicate key in metadata cache: {key}")
+        result[key] = value
+    return result
+
+
+def _loads_json(text: str):
+    return json.loads(text, object_pairs_hook=_reject_duplicate_keys)
+
+
+def _validated_record(app_id, record):
+    normalized_id = str(app_id)
+    if not isinstance(record, Mapping):
+        raise ValueError(
+            f"metadata cache record for app_id {normalized_id} must be a JSON object"
+        )
+    if "app_id" in record and str(record["app_id"]) != normalized_id:
+        raise ValueError(
+            f"metadata cache key {normalized_id} conflicts with record app_id {record['app_id']}"
+        )
+    app_name = record.get("app_name") or record.get("name") or record.get("title")
+    app_description = record.get("app_description") or record.get("description")
+    if not isinstance(app_name, str) or not app_name.strip():
+        raise ValueError(f"metadata cache record for app_id {normalized_id} is missing app_name")
+    if not isinstance(app_description, str) or not app_description.strip():
+        raise ValueError(
+            f"metadata cache record for app_id {normalized_id} is missing app_description"
+        )
+    return dict(record)
+
+
+def _add_record(cache, app_id, record):
+    normalized_id = str(app_id)
+    if normalized_id in cache:
+        raise ValueError(f"Duplicate app_id in metadata cache: {normalized_id}")
+    cache[normalized_id] = _validated_record(normalized_id, record)
+
+
+def load_app_metadata_cache(path):
+    """Return a validated app-id mapping from JSON, JSON-list, or JSONL input."""
+    if path is None:
+        return None
+
+    path = Path(path)
+    if path.suffix.lower() == ".jsonl":
+        cache = {}
+        with path.open(encoding="utf-8") as file:
+            for line_number, line in enumerate(file, start=1):
+                if not line.strip():
+                    continue
+                record = _loads_json(line)
+                if not isinstance(record, Mapping) or "app_id" not in record:
+                    raise ValueError(
+                        f"metadata JSONL record on line {line_number} must include app_id"
+                    )
+                _add_record(cache, record["app_id"], record)
+        return cache
+
+    loaded = _loads_json(path.read_text(encoding="utf-8"))
+    if isinstance(loaded, list):
+        cache = {}
+        for index, record in enumerate(loaded):
+            if not isinstance(record, Mapping) or "app_id" not in record:
+                raise ValueError(f"metadata JSON list record {index} must include app_id")
+            _add_record(cache, record["app_id"], record)
+        return cache
+    if not isinstance(loaded, Mapping):
+        raise ValueError(f"Metadata cache must be a JSON object, list, or JSONL: {path}")
+
+    cache = {}
+    for app_id, record in loaded.items():
+        _add_record(cache, app_id, record)
+    return cache
+
+
+def get_app_metadata(app_id, metadata_cache):
+    """Return the validated app name and description for one app ID."""
+    normalized_id = str(app_id)
+    record = metadata_cache.get(normalized_id)
+    if record is None:
+        raise KeyError(f"app_id {normalized_id} is missing from the app metadata cache")
+    record = _validated_record(normalized_id, record)
+    app_name = record.get("app_name") or record.get("name") or record.get("title")
+    app_description = record.get("app_description") or record.get("description")
+    return app_name, app_description

approach/config.py

@@ -0,0 +1,156 @@
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Mapping, Optional
+
+
+OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
+
+
+@dataclass(frozen=True)
+class ProviderConfig:
+    name: str
+    provider: str
+    model: str
+    base_url: str
+    api_key_env: str
+    router_options: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def api_key(self) -> str:
+        return os.environ.get(self.api_key_env, "")
+
+
+DEFAULT_PROFILES: Dict[str, ProviderConfig] = {
+    "default": ProviderConfig(
+        name="default",
+        provider="openrouter",
+        model=os.environ.get("OPENROUTER_MODEL", "openai/gpt-5.6-sol"),
+        base_url=OPENROUTER_BASE_URL,
+        api_key_env="OPENROUTER_API_KEY",
+    ),
+    "best_value": ProviderConfig(
+        name="best_value",
+        provider="openrouter",
+        model=os.environ.get("OPENROUTER_BEST_VALUE_MODEL", "openai/gpt-5.6-terra"),
+        base_url=OPENROUTER_BASE_URL,
+        api_key_env="OPENROUTER_API_KEY",
+    ),
+    "paper_openai": ProviderConfig(
+        name="paper_openai",
+        provider="openrouter",
+        model="openai/gpt-4o-2024-08-06",
+        base_url=OPENROUTER_BASE_URL,
+        api_key_env="OPENROUTER_API_KEY",
+    ),
+    "paper_claude": ProviderConfig(
+        name="paper_claude",
+        provider="openrouter",
+        model="anthropic/claude-3.5-sonnet",
+        base_url=OPENROUTER_BASE_URL,
+        api_key_env="OPENROUTER_API_KEY",
+    ),
+    "paper_gemini": ProviderConfig(
+        name="paper_gemini",
+        provider="openrouter",
+        model="google/gemini-pro-1.5",
+        base_url=OPENROUTER_BASE_URL,
+        api_key_env="OPENROUTER_API_KEY",
+    ),
+    "fastuse_experimental": ProviderConfig(
+        name="fastuse_experimental",
+        provider="openrouter",
+        model=os.environ.get("OPENROUTER_FASTUSE_MODEL", "google/gemini-3.1-pro-preview"),
+        base_url=OPENROUTER_BASE_URL,
+        api_key_env="OPENROUTER_API_KEY",
+    ),
+}
+
+
+PROFILE_ALIASES = {
+    "openrouter": "default",
+    "gpt4v": "paper_openai",
+    "gpt4v_abl": "paper_openai",
+    "claude35sonnet": "paper_claude",
+    "gemini15pro": "paper_gemini",
+    "gemini31pro": "fastuse_experimental",
+    "gemini-3-flash-preview-nothinking": "default",
+    "gemini-3-flash-preview-nothinking_abl": "default",
+}
+
+
+def _load_yaml_or_json(path: Path) -> Mapping[str, Any]:
+    text = path.read_text()
+    if path.suffix.lower() == ".json":
+        return json.loads(text)
+    try:
+        import yaml
+    except ImportError as exc:
+        raise RuntimeError("Install PyYAML or use a JSON config file.") from exc
+    loaded = yaml.safe_load(text)
+    return loaded or {}
+
+
+def _profile_from_mapping(name: str, values: Mapping[str, Any]) -> ProviderConfig:
+    return ProviderConfig(
+        name=name,
+        provider=values["provider"],
+        model=values["model"],
+        base_url=values.get("base_url", OPENROUTER_BASE_URL),
+        api_key_env=values.get("api_key_env", "OPENROUTER_API_KEY"),
+        router_options=dict(values.get("router_options", {})),
+    )
+
+
+def load_profiles(path: Optional[str] = None) -> Dict[str, ProviderConfig]:
+    profiles = dict(DEFAULT_PROFILES)
+    if not path:
+        return profiles
+
+    raw = _load_yaml_or_json(Path(path))
+    raw_profiles = raw.get("profiles", raw)
+    for name, values in raw_profiles.items():
+        profiles[name] = _profile_from_mapping(name, values)
+    return profiles
+
+
+def get_model_profile(name: str = "default", config_path: Optional[str] = None) -> ProviderConfig:
+    profile_name = PROFILE_ALIASES.get(name, name)
+    profiles = load_profiles(config_path)
+    if profile_name not in profiles:
+        valid = ", ".join(sorted(profiles))
+        raise ValueError(f"Unknown model profile '{name}'. Valid profiles: {valid}")
+    return profiles[profile_name]
+
+
+def validate_model_available(profile: ProviderConfig, models_catalog: Optional[Any] = None) -> bool:
+    if models_catalog is None:
+        return True
+    if isinstance(models_catalog, Mapping):
+        models_catalog = models_catalog.get("data", models_catalog.keys())
+    model_ids = {
+        entry.get("id") if isinstance(entry, Mapping) else str(entry)
+        for entry in models_catalog
+    }
+    if profile.model not in model_ids:
+        raise ValueError(f"Model {profile.model!r} was not found in the supplied catalog.")
+    return True
+
+
+def sanitize_manifest(
+    profile: ProviderConfig,
+    request_payload: Mapping[str, Any],
+    response_payload: Optional[Mapping[str, Any]] = None,
+) -> Dict[str, Any]:
+    response_payload = response_payload or {}
+    return {
+        "provider": profile.provider,
+        "profile": profile.name,
+        "request_model": request_payload.get("model"),
+        "response_model": response_payload.get("model"),
+        "base_url": profile.base_url,
+        "api_key_env": profile.api_key_env,
+        "router_options": dict(profile.router_options),
+        "usage": response_payload.get("usage", {}),
+    }

approach/method_gemini.py

@@ -0,0 +1,655 @@
+import os
+import re
+import sys
+import json
+import time
+import random
+import string
+import argparse
+import pandas as pd
+import copy
+import logging
+from tqdm import tqdm
+import PIL
+from PIL import ImageFile
+PIL.ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(BASE_DIR)
+from approach.pipeline_utils import (
+    enrich_ape_results,
+    output_path_for_selection,
+    select_jsonl_lines,
+    write_json_atomic,
+)
+
+os.environ['TORCH_HOME'] = './.cache'
+os.environ['HF_HOME'] = './.cache'
+
+
+vlm = 'llava7b'
+llm = 'gpt_3.5_turbo'
+ovod = 'grounding_dino'
+
+vlm_prompt = ''
+
+### Configurable
+host_device = os.getenv('ORIENTER_LEGACY_HOST', 'local')
+perspective = 'all_perspective'
+# perspective = 'direct_back'
+# perspective = 'direct_front'
+# perspective = 'direct_side'
+# perspective = 'direct_top'
+# perspective = 'eyelevel'
+# perspective = 'overlook'
+### Configurable
+
+llava_path = os.path.join(BASE_DIR, 'approach/vlm/LLaVA/llava/eval')
+grounding_dino_path = os.path.join(BASE_DIR, 'approach/ovod/GroundingDINO')
+ape_path = os.path.join(BASE_DIR, 'approach/ovod/APE')
+# TODO: data on the CUHK server
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_merged/images/interactable')
+# icse
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_merged/images/test_set_merged')
+# icse_rebuttal
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_angleview/images/all_perspective')
+# fse
+images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_det/images/union3')
+
+
+# def generate_grounding_dino_command(ovod_image_path, ovod_output_dir, ovod_candidates):
+#     gdino_object_str = ' . '.join(ovod_candidates)
+
+#     gdino_command = f'''
+#     python demo/inference_on_a_image.py \
+#         -c groundingdino/config/GroundingDINO_SwinT_OGC.py \
+#         -p weights/groundingdino_swint_ogc.pth \
+#         -i {ovod_image_path} \
+#         -o "{ovod_output_dir}" \
+#         -t "{gdino_object_str}"
+#     '''
+#     return gdino_command
+
+
+def generate_question_file(img_folder, dst_path):
+    # Ensure the directory exists
+    if not os.path.exists(img_folder):
+        print(f"Error: Directory {img_folder} does not exist.")
+        return
+
+    # List all files in the directory
+    all_files = os.listdir(img_folder)
+
+    # Filter out files that are not images (based on extension). You can add more if needed.
+    image_extensions = [".jpg", ".jpeg", ".png", ".bmp", ".gif", ".tiff"]
+    image_files = [f for f in all_files if any(f.lower().endswith(ext) for ext in image_extensions)]
+
+    # Open the output file for writing
+    with open(dst_path, 'w') as out_file:
+        for index, img_file in enumerate(image_files):
+            data = {
+                "question_id": index,
+                "image": img_file,
+                "text": vlm_prompt,
+                "category": "detail"
+            }
+            out_file.write(json.dumps(data) + '\n')
+
+    print(f"Processed {len(image_files)} images. Output saved to {dst_path}.")
+
+
+def generate_answer_id(length=20):
+    # Define the characters that can be used in the string
+    characters = string.ascii_letters + string.digits
+    # Generate a random string of the specified length
+    answer_id = ''.join(random.choice(characters) for _ in range(length))
+    return answer_id
+
+
+def method(vlm=vlm, llm=llm, ovod=ovod, start_index=None, end_index=None, shard_index=None, num_shards=None):
+    # vlm_question_path = os.path.join(BASE_DIR, 'approach/vlm/sampled_vlm_questions.jsonl')
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_vlm_questions_from671.jsonl')
+    # icse
+    # complete_vlm_question_abl_i_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_complete_vlm_questions_ablation_interactability.jsonl')
+
+    # icse
+    # complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_complete_vlm_questions.jsonl')
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/gpt4v_failure_questions.jsonl')
+    # icse_rebuttal
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_vlm_questions.jsonl')
+    # complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_complete_vlm_questions.jsonl')
+    # fse
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/fse_union3_{vlm}_questions.jsonl')
+    complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_union3_complete_{vlm}_questions.jsonl')
+    vlm_question_path = complete_vlm_question_path
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/gpt4v_failure_questions.jsonl')
+    # playground/data/coco2014_val_qa_eval/qa90_questions.jsonl
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_slowerspeed_1.jsonl')
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_higherspeed_r3_1_from195.jsonl')
+
+    # icse
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/cuhk_icse_test_set_merged_{vlm}_answer.jsonl')
+    # llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer.jsonl')
+    # gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_ablation_no_interactability_r1_0.jsonl')
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_cuhk_icse_test_set_merged_gpt4v_answer_higherspeed_r3_0.jsonl')
+    # interactability_abl_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_hit_icse_test_set_merged_gemini_answer_ablation_no_interactability_r1_0.jsonl')
+    # icse_rebuttal
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # fse
+    vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_uninon3_{vlm}_answer.jsonl')
+    llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_fse_uninon3_{vlm}_answer.jsonl')
+    gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_union3_{vlm}_answer.jsonl')
+
+    # icse
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_icse_rbt_{perspective}_gpt4v_answer.jsonl')
+    # icse_rebuttal
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_{vlm}_answer_265_528.jsonl')
+    # fse
+    gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_uninon3_gpt4v_answer.jsonl')
+    # print(gpt4_results_answer_path)
+    # interactability_abl_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_icse_rbt_{perspective}_answer.jsonl')
+
+    # icse
+    # # /path/to/answer-file-our.jsonl
+    # # llm_candidate_path = os.path.join(BASE_DIR, f'approach/llm/{vlm}_{llm}_cancidate_objects.json')
+    # interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/icse_test_set_merged_{vlm}_{llm}_interactable_objects.json')
+    # ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/icse_test_set_merged_{ovod}')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_object_bbox_gpu3_2.json')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_interactability_object_bbox_gpu0.json')
+    # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_feedback_object_bbox_gpu2.json')
+
+    # icse_rebuttal
+    # interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/icse_rbt_{perspective}_{vlm}_{llm}_interactable_objects.json')
+    # ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/icse_rbt_{perspective}_{ovod}')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_object_bbox_gpu3_2.json')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_interactability_object_bbox_gpu0.json')
+    # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_rbt_{perspective}_{vlm}_{llm}_{ovod}_object_bbox_gpu3_265_528.json')
+
+    gpu = '3'
+    # fse
+    interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/realfse_union3_{vlm}_{llm}_interactable_objects.json') # TODO: Tentatively unused
+    ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/realfse_union3_{ovod}')
+    # GPU 0
+    oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/realfse_union3_{vlm}_{ovod}_object_bbox_gpu{gpu}.json')
+    oovd_object_bbox_path = output_path_for_selection(
+        oovd_object_bbox_path,
+        start_index=start_index,
+        end_index=end_index,
+        shard_index=shard_index,
+        num_shards=num_shards,
+    )
+
+    llava7b_model_path = os.path.join(BASE_DIR, 'approach/vlm/LLaVA/checkpoints/llava-v1.5-7b')
+    llava7b_cmd = f'''
+    python model_vqa.py \
+        --model-path {llava7b_model_path} \
+        --question-file \
+        {complete_vlm_question_path} \
+        --image-folder \
+        {images_dir} \
+        --answers-file \
+        {llava_vlm_answer_path}
+    '''
+
+    # STEP #0
+    # First time of running
+    # TODO: Check whether it is first-time running
+    # generate_question_file(images_dir, vlm_question_path)
+
+    # STEP #1
+    # VLM - Get local context
+    # print(f'VLM {vlm} analysis begins ...')
+    # vlm_start_time = time.time()
+    # logging.info(f'STEP #1 VLM {vlm} analysis started at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    # if vlm == 'llava7b':
+    #     original_path = os.path.dirname(__file__)
+    #     os.chdir(llava_path)
+    #     os.system(llava7b_cmd)
+    #     os.chdir(original_path)
+    # elif vlm == 'generate_q_file':
+    #     from approach.vlm.gpt4v.gpt4v import process_image_q
+    #     with open(vlm_question_path, 'r') as q_file, open(complete_vlm_question_path, 'w') as cq_file:  # Open answer file in append mode
+    #         qfile_lines = q_file.readlines()
+    #         key_idx = 0
+    #         for line in tqdm(qfile_lines):
+    #             key_idx = (key_idx + 1) % 4
+
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             if vlm == 'gpt4v_abl':
+    #                 gpt4v_ablation = True
+    #             elif vlm == 'gpt4v':
+    #                 gpt4v_ablation = False
+    #             gpt4v_q = process_image_q(image_question, image_path, key_idx) 
+    #             data = {
+    #                 "question_id": line_data['question_id'],
+    #                 "image": line_data['image'],
+    #                 "text": gpt4v_q,
+    #                 "category": "detail"
+    #             }
+    #             cq_file.write(json.dumps(data) + '\n')       
+    # elif vlm == 'gpt4v' or vlm == 'gpt4v_abl' or vlm == 'claude35sonnet' or vlm == 'gemini15pro':
+    #     from approach.vlm.gpt4v.gpt4v import process_image
+    #     with open(vlm_question_path, 'r') as q_file, open(vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+    #         qfile_lines = q_file.readlines()
+    #         key_idx = 0
+    #         for line in tqdm(qfile_lines):
+    #             key_idx = (key_idx + 1) % 4
+
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             gpt4v_ablation = False
+    #             if vlm == 'gpt4v_abl':
+    #                 gpt4v_ablation = True
+    #             elif vlm == 'gpt4v':
+    #                 gpt4v_ablation = False
+    #             gpt4v_res = process_image(vlm, image_question, image_path, gpt4v_ablation, key_idx)
+    #             print(gpt4v_res)
+    #             # time.sleep(3)
+
+    #             ans_item = {
+    #                 "question_id": line_data['question_id'], 
+    #                 "prompt": '', 
+    #                 "text": gpt4v_res, 
+    #                 "answer_id": generate_answer_id(), 
+    #                 "model_id": vlm, 
+    #                 "metadata": {}
+    #             }
+
+    #             a_file.write(json.dumps(ans_item) + '\n')  
+    #             a_file.flush()
+    # elif vlm == 'bing':
+    #     from approach.vlm.bing.bing import context_conversation
+    #     with open(vlm_question_path, 'r') as q_file, open(vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+    #         for line in tqdm(q_file):
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             bing_res = context_conversation(image_question, image_path)
+    #             print(bing_res)
+    #             time.sleep(15)
+
+    #             ans_item = {
+    #                 "question_id": line_data['question_id'], 
+    #                 "prompt": image_question, 
+    #                 "text": bing_res, 
+    #                 "answer_id": generate_answer_id(), 
+    #                 "model_id": vlm, 
+    #                 "metadata": {}
+    #             }
+
+    #             a_file.write(json.dumps(ans_item) + '\n')  
+    # elif vlm == 'gemini':
+    #     import pathlib
+    #     import textwrap
+
+    #     import google.generativeai as genai
+
+    #     GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "[GOOGLE_API_KEY]")
+    #     genai.configure(api_key=GOOGLE_API_KEY)
+
+    #     safety_settings = [
+    #         {
+    #             "category": "HARM_CATEGORY_DANGEROUS",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_HARASSMENT",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_HATE_SPEECH",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #     ]
+
+    #     # before fse
+    #     # model = genai.GenerativeModel('gemini-pro-vision')
+    #     # fse
+    #     model = genai.GenerativeModel('gemini-1.5-pro')
+
+    #     with open(vlm_question_path, 'r') as q_file, open(gemini_vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+    #         qfile_lines = q_file.readlines()[0:2]
+    #         for line in tqdm(qfile_lines):
+
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             gemini_response = model.generate_content([image_question, PIL.Image.open(image_path)], safety_settings=safety_settings)
+    #             gemini_response.resolve()
+    #             try:
+    #                 gemini_response = gemini_response.text
+    #                 ans_item = {
+    #                     "question_id": line_data['question_id'], 
+    #                     "prompt": image_question, 
+    #                     "text": gemini_response, 
+    #                     "answer_id": generate_answer_id(), 
+    #                     "model_id": vlm, 
+    #                     "metadata": {}
+    #                 }
+
+    #                 a_file.write(json.dumps(ans_item) + '\n')  
+    #                 a_file.flush()
+    #             except Exception as e:
+    #                 print(f"Error for image {image_path}. Error: {e}")                
+    #             # print(gemini_response)
+    #             time.sleep(30)
+
+
+
+    # elif vlm.endswith('_pass'):
+    #     print(f'Passing VLM {vlm} ...')
+    # else:
+    #     raise Exception('Unrecognized VLM!')
+
+    # logging.info(f'STEP #1 VLM {vlm} analysis completed at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    # logging.info(f'STEP #1 VLM {vlm} analysis total time taken: {time.time() - vlm_start_time} seconds')
+
+
+
+
+
+    # Get label candidates
+    # if llm == 'gpt_3.5_turbo':
+    #     from approach.llm.gpt_polling import infer_object_candidates
+    # elif llm == 'llama2':
+    #     from approach.llm.llama import infer_object_candidates
+
+
+    # STEP #3
+    # if llm == 'gpt_3.5_turbo' or llm == 'gpt_3.5_turbo_abl':
+    #     from approach.llm.gpt_polling import infer_objects
+
+    #     if llm == 'gpt_3.5_turbo_abl':
+    #         gpt35_ablation = True
+    #     elif llm == 'gpt_3.5_turbo':
+    #         gpt35_ablation = False
+    #     infer_objects(vlm_question_path, vlm_answer_path, interactable_object_path, gpt35_ablation)
+    # elif llm == 'llama2':
+    #     from approach.llm.llama import infer_objects
+    #     exit(0)
+    # elif llm.endswith('_pass'):
+    #     print(f'Passing LLM {llm} ...')
+    # else:
+    #     raise Exception('Unrecognized LLM!')
+
+
+
+
+
+
+    # STEP #2
+    # Open-vocabulary object detection
+    ovod_start_time = time.time()
+    logging.info(f'STEP #2 OVOD {ovod} analysis started at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    vlm_questions = {}
+    with open(complete_vlm_question_path, 'r') as q_file:
+        for line in q_file:
+            line_data = json.loads(line)
+            vlm_questions[line_data['question_id']] = line_data
+            # print(line_data['question_id'])
+
+    if ovod == 'grounding_dino':
+        from approach.ovod.GroundingDINO.demo.inference_on_a_image import process_grounding_dino
+
+        original_path = os.path.dirname(__file__)
+        os.chdir(grounding_dino_path)
+
+        # with open(llm_candidate_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        with open(interactable_object_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+            llm_candidate = json.load(llm_candidate_file)
+            all_oovd_res = {}
+            for image_index in llm_candidate.keys():
+                print(image_index)
+                 # TODO: 
+                if (image_index != '1160'):
+                    ovod_image_path = os.path.join(images_dir, vlm_questions[int(image_index)]["image"])
+                    ovod_candidates = llm_candidate[image_index]['interactable_objects']
+                    # grounding_dino_command = generate_grounding_dino_command(ovod_image_path, ovod_output_dir, ovod_candidates)
+                    # print(grounding_dino_command)
+                    # os.system(grounding_dino_command)
+
+                    gdino_res = process_grounding_dino(
+                        config_file='groundingdino/config/GroundingDINO_SwinT_OGC.py', 
+                        checkpoint_path='weights/groundingdino_swint_ogc.pth', 
+                        image_path=ovod_image_path, 
+                        ovod_candidates=ovod_candidates, 
+                        output_dir=ovod_output_dir, 
+                        box_threshold=0.3, 
+                        text_threshold=0.25, 
+                        token_spans=None, 
+                        cpu_only=False
+                    )
+                    # print(gdino_res)
+
+                    object_oovd_item = copy.deepcopy(llm_candidate[image_index])
+                    object_oovd_item['oovd_result'] = gdino_res
+                    print(object_oovd_item)
+                    all_oovd_res[image_index] = object_oovd_item
+
+            json.dump(all_oovd_res, oovd_file, indent=4)
+
+
+        os.chdir(original_path)
+
+    elif ovod == 'ape_d' or ovod == 'ape_d_abl':
+        from approach.ovod.APE.demo.ape_inference import run_ape_model_inference
+
+        original_path = os.path.dirname(__file__)
+        os.chdir(ape_path)
+
+        # with open(llm_candidate_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+
+        all_ape_res = []
+        # General
+        with open(vlm_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        # GPT-4 eval
+        # with open(gpt4_results_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        # Gemini abl i eval
+        # with open(interactability_abl_results_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+
+            # llm_candidate = json.load(llm_candidate_file)
+            all_oovd_res = {}
+
+
+            # candidate_lines = llm_candidate_file.readlines()[265:528]
+            # GPU 0
+            # candidate_lines = llm_candidate_file.readlines()[0:230]
+            # candidate_lines = llm_candidate_file.readlines()[230:460]
+            # candidate_lines = llm_candidate_file.readlines()[460:690]
+            candidate_lines = select_jsonl_lines(
+                llm_candidate_file.readlines(),
+                start_index=start_index,
+                end_index=end_index,
+                shard_index=shard_index,
+                num_shards=num_shards,
+            )
+            # candidate_lines = llm_candidate_file.readlines()
+            for line in tqdm(candidate_lines):
+                ans_item = {}
+
+                line_data = json.loads(line)
+
+            # for image_index in llm_candidate.keys():
+                # print(image_index)
+                 # TODO: 
+                # if (image_index != '1160'):
+                # image_index = line_data['question_id']
+
+                image_name = vlm_questions[line_data['question_id']]["image"]
+                ovod_image_path = os.path.join(images_dir, image_name)
+
+                # General
+                # try:
+                #     if line_data['text'].startswith(" ```json"):
+                #         ovod_candidates = json.loads(line_data['text'][8:-4])['objects']
+                #     elif line_data['text'].startswith(' {\"objects\"'):
+                #         ovod_candidates = json.loads(line_data['text'])['objects']
+                #     else:
+                #         print(f"Error for decoding IVO json for {image_name}.")
+                # except Exception as e:
+                #     print(f"Error for decoding IVO json for image {image_name}. Error: {e}")
+                #     continue
+
+
+                # GPT-4v
+                if line_data['text']:
+                    ovod_candidates = line_data['text']['objects']
+                else:
+                    continue
+
+
+                all_res = []
+                for ocd in ovod_candidates.keys():
+                    referring_expr_str = ovod_candidates[ocd]
+
+                    translator = str.maketrans('', '', string.punctuation)
+                    if type(referring_expr_str) is str:
+                        referring_expr_str = referring_expr_str.translate(translator)
+                    else:
+                        # dict
+                        referring_expr_str = ' '.join(referring_expr_str.values())
+                        referring_expr_str = referring_expr_str.translate(translator)
+                    referring_expr_str = f'{ocd}: {referring_expr_str}'
+                    # print(image_name, referring_expr_str)
+                    all_res.append(referring_expr_str)
+
+                if ovod == 'ape_d':
+                    ape_threshold = 0.15
+                elif ovod == 'ape_d_abl':
+                    ape_threshold = 0.1
+
+                ape_res = []
+                try:
+                    ape_res = run_ape_model_inference(
+                        config_file='configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py',
+                        input_path=ovod_image_path,
+                        # GPU 0
+                        output_path=f'./realfse_{vlm}_{ovod}_gpu0123',
+                        confidence_threshold=ape_threshold,
+                        text_prompt=', '.join(all_res),
+                        with_box=True,
+                        with_mask=False,
+                        with_sseg=False,
+                        opts=[
+                                "train.init_checkpoint='./ape_d_model_final.pth'",
+                                "model.model_language.cache_dir=''",
+                                "model.model_vision.select_box_nums_for_evaluation=500",
+                                "model.model_vision.text_feature_bank_reset=True",
+                                "model.model_vision.backbone.net.xattn=False",
+                                "model.model_vision.transformer.encoder.pytorch_attn=True",
+                                "model.model_vision.transformer.decoder.pytorch_attn=True"
+                        ]
+                    )
+                except Exception as e:
+                    print(f"Error for image {image_name}. Error: {e}")
+
+                all_ape_res.extend(enrich_ape_results(ape_res, image_name, extract_image_id))
+                # object_oovd_item = copy.deepcopy(llm_candidate[image_index])
+                # object_oovd_item['ape_result'] = all_ape_res
+                # # print(object_oovd_item)
+                # all_oovd_res[image_index] = object_oovd_item
+
+            write_json_atomic(oovd_object_bbox_path, all_ape_res)
+
+
+        os.chdir(original_path)                    
+
+    elif ovod.endswith('_pass'):
+        print(f'Passing OVOD {ovod} ...')
+    else:
+        raise Exception('Unrecognized OVOD!')
+
+    logging.info(f'STEP #2 OVOD {ovod} analysis completed at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    logging.info(f'STEP #2 OVOD {ovod} analysis total time taken: {time.time() - ovod_start_time} seconds')
+
+# icse & fse
+def extract_image_id(image_name):
+    # Split the image name into parts and form the image_id accordingly
+    parts = image_name.split('_')
+    print(image_name)
+    base, extension = parts[1].split('.')
+    return int(parts[0] + base.zfill(3))
+# icse_rebuttal
+# def extract_image_id(image_name):
+#     # Split the image name into parts by underscore
+#     parts = image_name.split('_')
+#     # Extracting the first part as the base and the numeric portion of the third part before the file extension
+#     base = parts[0]  # This will give '625470'
+#     numeric_part = parts[2].split('.')[0]  # This will give 'b3'
+#     # Removing non-numeric characters from 'b3'
+#     numeric_part = ''.join(filter(str.isdigit, numeric_part))
+#     # Zfill is used to ensure the numeric part has at least 3 digits, then combining with base
+#     print(int(base + numeric_part.zfill(3)))
+#     return int(base + numeric_part.zfill(3))
+
+
+def main():
+    vlms = ['llava7b', 'bing', 'gpt4v']
+    llms = ['llama2', 'gpt_3.5_turbo']
+    ovods = ['grounding_dino', 'glip', 'ape_d']
+
+
+# python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d
+# python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d_abl
+# python method.py -v claude35sonnet -l gpt_3.5_turbo -o ape_d
+# python method.py -v gemini -l gpt_3.5_turbo -o ape_d
+if __name__=='__main__':
+    # main()
+
+    parser = argparse.ArgumentParser("Method", add_help=True)
+    parser.add_argument("--vlm", "-v", type=str, required=True, help="vlm")
+    parser.add_argument("--llm", "-l", type=str, required=True, help="llm")
+    parser.add_argument("--ovod", "-o", type=str, required=True, help="ovod")
+    parser.add_argument("--start-index", type=int, default=None, help="first JSONL row to process")
+    parser.add_argument("--end-index", type=int, default=None, help="exclusive JSONL row end")
+    parser.add_argument("--shard-index", type=int, default=None, help="zero-based shard index")
+    parser.add_argument("--num-shards", type=int, default=None, help="total number of shards")
+    args = parser.parse_args()
+
+    gpu = '3'
+    stage = '2'
+    idx = '0001'
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+        datefmt='%m/%d/%Y %H:%M:%S',
+        level=logging.INFO,
+        # gpu
+        filename=f'../log/realfse/240910_{args.vlm}_{stage}_g{gpu}_{idx}.log'
+        )
+
+    method(
+        args.vlm,
+        args.llm,
+        args.ovod,
+        start_index=args.start_index,
+        end_index=args.end_index,
+        shard_index=args.shard_index,
+        num_shards=args.num_shards,
+    )
+
+# CUDA_VISIBLE_DEVICES=0 python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d > ../log/realfse/240910_gpt4v_2_g0_0001.txt
+# CUDA_VISIBLE_DEVICES=3 python method_gemini.py -v gemini15pro -l gpt_3.5_turbo -o ape_d > ../log/realfse/240910_gemini15pro_2_g3_0001.txt

approach/providers.py

@@ -0,0 +1,149 @@
+import json
+from dataclasses import dataclass
+from urllib import request
+from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Union
+
+from approach.config import ProviderConfig
+
+
+PostFn = Callable[..., Any]
+
+
+@dataclass(frozen=True)
+class EncodedImage:
+    data: str
+    media_type: str = "image/jpeg"
+
+
+class UrlLibResponse:
+    def __init__(self, status: int, body: bytes):
+        self.status_code = status
+        self._body = body
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise RuntimeError(f"HTTP request failed with status {self.status_code}")
+
+    def json(self):
+        return json.loads(self._body.decode("utf-8"))
+
+
+def default_post(url: str, headers: Mapping[str, str], json: Mapping[str, Any], timeout: int):
+    req = request.Request(
+        url,
+        data=json_dumps_bytes(json),
+        headers=dict(headers),
+        method="POST",
+    )
+    with request.urlopen(req, timeout=timeout) as response:
+        return UrlLibResponse(response.status, response.read())
+
+
+def json_dumps_bytes(payload: Mapping[str, Any]) -> bytes:
+    return json.dumps(payload).encode("utf-8")
+
+
+def make_multimodal_content(
+    text: str,
+    image_b64s: Optional[Iterable[Union[str, EncodedImage]]] = None,
+) -> List[Dict[str, Any]]:
+    content = [{"type": "text", "text": text}]
+    for image in image_b64s or []:
+        encoded = image if isinstance(image, EncodedImage) else EncodedImage(data=image)
+        content.append(
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:{encoded.media_type};base64,{encoded.data}"
+                },
+            }
+        )
+    return content
+
+
+class OpenAICompatibleChatClient:
+    def __init__(self, profile: ProviderConfig, post: Optional[PostFn] = None):
+        self.profile = profile
+        self.post = post or default_post
+
+    def build_payload(
+        self,
+        prompt: str,
+        image_b64s: Optional[Iterable[Union[str, EncodedImage]]] = None,
+        response_format: Optional[Mapping[str, Any]] = None,
+        temperature: float = 0,
+        max_tokens: int = 4096,
+    ) -> Dict[str, Any]:
+        return self.build_messages_payload(
+            [
+                {
+                    "role": "user",
+                    "content": make_multimodal_content(prompt, image_b64s),
+                }
+            ],
+            response_format=response_format,
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+
+    def build_messages_payload(
+        self,
+        messages: Iterable[Mapping[str, Any]],
+        response_format: Optional[Mapping[str, Any]] = None,
+        temperature: float = 0,
+        max_tokens: int = 4096,
+    ) -> Dict[str, Any]:
+        payload: Dict[str, Any] = {
+            "model": self.profile.model,
+            "temperature": temperature,
+            "messages": [dict(message) for message in messages],
+            "max_tokens": max_tokens,
+        }
+        if response_format:
+            payload["response_format"] = dict(response_format)
+        if self.profile.router_options:
+            payload["provider"] = dict(self.profile.router_options)
+        return payload
+
+    def chat_completion(self, payload: Mapping[str, Any]) -> Mapping[str, Any]:
+        if not self.profile.api_key:
+            raise RuntimeError(f"Missing API key in environment variable {self.profile.api_key_env}")
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.profile.api_key}",
+        }
+        response = self.post(
+            f"{self.profile.base_url.rstrip('/')}/chat/completions",
+            headers=headers,
+            json=dict(payload),
+            timeout=120,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def complete_json(
+        self,
+        prompt: str,
+        image_b64s: Optional[Iterable[Union[str, EncodedImage]]] = None,
+        response_format: Optional[Mapping[str, Any]] = None,
+    ) -> Any:
+        payload = self.build_payload(prompt, image_b64s, response_format=response_format)
+        return self.complete_json_payload(payload)
+
+    def complete_json_messages(
+        self,
+        messages: Iterable[Mapping[str, Any]],
+        response_format: Optional[Mapping[str, Any]] = None,
+    ) -> Any:
+        payload = self.build_messages_payload(
+            messages,
+            response_format=response_format,
+        )
+        return self.complete_json_payload(payload)
+
+    def complete_json_payload(self, payload: Mapping[str, Any]) -> Any:
+        response = self.chat_completion(payload)
+        content = response["choices"][0]["message"]["content"].strip()
+        if content.startswith("```"):
+            content = content.strip("`").split("\n", 1)[-1]
+        return json.loads(content)

approach/reflection.py

@@ -0,0 +1,227 @@
+import io
+import base64
+import json
+import math
+from dataclasses import dataclass
+from typing import Callable, Dict, Iterable, List, Sequence
+
+from PIL import Image, ImageDraw
+
+from approach.config import get_model_profile
+from approach.providers import EncodedImage, OpenAICompatibleChatClient
+
+
+Detection = Dict[str, object]
+Advisor = Callable[..., Dict[str, object]]
+Miner = Callable[[List[Dict[str, object]]], Sequence[str]]
+Detector = Callable[[Sequence[str], List[Detection]], List[Detection]]
+
+
+PII5_PROMPT = (
+    "PII.5 Mirroring-Based Regional Verification. Compare the original VR scene "
+    "and the cropped detected element. Verify whether the crop is a correct "
+    "user-interactable game element and whether it violates XR spatial constraints."
+)
+PII6_PROMPT = (
+    "PII.6 Unsuccessful Detection Reflection. Inspect the original scene with "
+    "current detection boxes and identify missing user-interactable CDs."
+)
+PII7_PROMPT = (
+    "PII.7 Advisor Feedback. Return JSON with verified, needs_refinement, and feedback. "
+    "Each feedback item must be a concise new or revised CD referring expression that "
+    "the detector can ground in the original scene."
+)
+
+
+@dataclass(frozen=True)
+class VerificationInput:
+    prompt: str
+    original_image: Image.Image
+    crop_image: Image.Image
+    detection: Detection
+
+
+def crop_detection(image: Image.Image, bbox: Sequence[float]) -> Image.Image:
+    if len(bbox) != 4:
+        raise ValueError("bbox must contain [x, y, width, height]")
+    x, y, width, height = bbox
+    left = max(0, math.floor(x))
+    top = max(0, math.floor(y))
+    right = min(image.width, math.ceil(x + width))
+    bottom = min(image.height, math.ceil(y + height))
+    if right <= left or bottom <= top:
+        raise ValueError(f"invalid bbox after clipping: {bbox}")
+    return image.crop((left, top, right, bottom))
+
+
+def make_successful_detection_request(image: Image.Image, detection: Detection) -> VerificationInput:
+    return VerificationInput(
+        prompt=PII5_PROMPT,
+        original_image=image,
+        crop_image=crop_detection(image, detection["bbox"]),
+        detection=detection,
+    )
+
+
+def draw_detections(image: Image.Image, detections: Iterable[Detection]) -> Image.Image:
+    annotated = image.copy()
+    draw = ImageDraw.Draw(annotated)
+    for idx, detection in enumerate(detections, start=1):
+        x, y, width, height = detection["bbox"]
+        box = (x, y, x + width, y + height)
+        draw.rectangle(box, outline="red", width=2)
+        draw.text((x, y), str(detection.get("category_name", idx)), fill="red")
+    return annotated
+
+
+def image_to_png_bytes(image: Image.Image) -> bytes:
+    buffer = io.BytesIO()
+    image.save(buffer, format="PNG")
+    return buffer.getvalue()
+
+
+def image_to_base64(image: Image.Image) -> str:
+    return base64.b64encode(image_to_png_bytes(image)).decode("utf-8")
+
+
+REFLECTION_RESPONSE_FORMAT = {
+    "type": "json_schema",
+    "json_schema": {
+        "name": "orienter_reflection",
+        "strict": True,
+        "schema": {
+            "type": "object",
+            "properties": {
+                "verified": {"type": "array", "items": {"type": "integer"}},
+                "needs_refinement": {"type": "array", "items": {"type": "integer"}},
+                "feedback": {"type": "array", "items": {"type": "string"}},
+            },
+            "required": ["verified", "needs_refinement", "feedback"],
+            "additionalProperties": False,
+        },
+    },
+}
+
+
+def build_reflection_prompt(
+    verification_requests: Sequence[VerificationInput],
+    pii5_prompt: str,
+    pii6_prompt: str,
+    pii7_prompt: str,
+) -> str:
+    detections = [
+        {
+            "index": idx,
+            "bbox": request.detection.get("bbox"),
+            "category_name": request.detection.get("category_name"),
+            "score": request.detection.get("score", request.detection.get("probability")),
+        }
+        for idx, request in enumerate(verification_requests)
+    ]
+    return (
+        f"{pii5_prompt}\n"
+        "For each detected CD, compare the original scene image with its cropped region image. "
+        "Mark false positives or boxes that do not match the intended interactable element as needs_refinement.\n\n"
+        f"{pii6_prompt}\n"
+        "Also inspect the annotated full-scene image for missing interactable CDs or wrong boxes.\n\n"
+        f"{pii7_prompt}\n"
+        "Return only JSON. Images are ordered as: (1) the boxed full scene for PII.6, "
+        "then (2) one original-scene and bbox-crop pair per detection, in detection-index order.\n\n"
+        f"Detections:\n{json.dumps(detections, ensure_ascii=True, indent=2)}"
+    )
+
+
+def openai_compatible_advisor(model_profile: str = "default") -> Advisor:
+    profile = get_model_profile(model_profile)
+    client = OpenAICompatibleChatClient(profile)
+
+    def advisor(**kwargs) -> Dict[str, object]:
+        verification_requests = kwargs["verification_requests"]
+        unsuccessful_detection_image = kwargs["unsuccessful_detection_image"]
+        image_b64s = [
+            EncodedImage(image_to_base64(unsuccessful_detection_image), "image/png")
+        ]
+        for request in verification_requests:
+            image_b64s.append(
+                EncodedImage(image_to_base64(request.original_image), "image/png")
+            )
+            image_b64s.append(
+                EncodedImage(image_to_base64(request.crop_image), "image/png")
+            )
+        prompt = build_reflection_prompt(
+            verification_requests,
+            kwargs["pii5_prompt"],
+            kwargs["pii6_prompt"],
+            kwargs["pii7_prompt"],
+        )
+        return client.complete_json(
+            prompt,
+            image_b64s,
+            response_format=REFLECTION_RESPONSE_FORMAT,
+        )
+
+    return advisor
+
+
+def run_reflection_loop(
+    image: Image.Image,
+    detections: List[Detection],
+    miner: Miner,
+    detector: Detector,
+    advisor: Advisor,
+    max_iterations: int = 10,
+) -> Dict[str, object]:
+    if max_iterations <= 0:
+        raise ValueError("max_iterations must be positive")
+
+    current_detections = list(detections)
+    trace = []
+    max_reached = False
+
+    for iteration in range(1, max_iterations + 1):
+        verification_requests = [
+            make_successful_detection_request(image, detection)
+            for detection in current_detections
+        ]
+        unsuccessful_scene = draw_detections(image, current_detections)
+        advice = advisor(
+            verification_requests=verification_requests,
+            unsuccessful_detection_image=unsuccessful_scene,
+            pii5_prompt=PII5_PROMPT,
+            pii6_prompt=PII6_PROMPT,
+            pii7_prompt=PII7_PROMPT,
+        )
+        trace.append(
+            {
+                "iteration": iteration,
+                "verified": advice.get("verified", []),
+                "needs_refinement": advice.get("needs_refinement", []),
+                "feedback": advice.get("feedback", []),
+            }
+        )
+
+        feedback = advice.get("feedback", [])
+        needs_refinement = advice.get("needs_refinement", [])
+        if not feedback and not needs_refinement:
+            break
+
+        rejected_indices = {
+            index
+            for index in needs_refinement
+            if isinstance(index, int) and 0 <= index < len(current_detections)
+        }
+        retained_detections = [
+            detection
+            for index, detection in enumerate(current_detections)
+            if index not in rejected_indices
+        ]
+        candidates = list(miner(trace))
+        current_detections = detector(candidates, retained_detections)
+    else:
+        max_reached = True
+
+    return {
+        "detections": current_detections,
+        "trace": trace,
+        "max_iterations_reached": max_reached,
+    }

approach/requirements.txt

@@ -0,0 +1,79 @@
+addict==2.4.0
+aiohttp==3.8.6
+APScheduler==3.10.0
+av==11.0.0
+beautifulsoup4==4.12.2
+BingImageCreator==0.5.0
+black==23.12.1
+caffe2==0.8.1
+certifi==2023.7.22
+cityscapesscripts==2.2.2
+click==8.0.1
+clip==0.2.0
+cloudpickle==3.0.0
+deepspeed==0.9.5
+docutils==0.20.1
+einops==0.7.0
+emoji==2.9.0
+fairscale==0.4.13
+fastapi==0.108.0
+fiftyone==0.23.2
+fire==0.5.0
+flash_attn==2.4.2
+Flask==2.1.1
+fvcore==0.1.5.post20221221
+gradio==4.12.0
+gunicorn==19.9.0
+httpx==0.26.0
+huggingface_hub==0.17.3
+hydra-core==1.3.2
+iopath==0.1.10
+ipdb==0.13.13
+lxml==4.9.2
+matplotlib==3.8.0
+mmcv==2.1.0
+mmdet==3.2.0
+nltk==3.8.1
+numpy==1.26.2
+omegaconf==2.3.0
+onnx==1.15.0
+openai==0.28.1
+pandas==2.1.4
+peft==0.4.0
+Pillow==10.0.1
+Pillow==10.1.0
+prompt_toolkit==3.0.43
+psutil==5.9.5
+pycocotools==2.0.7
+Pygments==2.16.1
+pytorch_lightning==2.1.3
+PyYAML==6.0.1
+PyYAML==6.0.1
+ray==2.9.0
+recommonmark==0.7.1
+redis==3.5.3
+Requests==2.31.0
+rich==13.7.0
+scipy==1.11.4
+selenium==4.16.0
+setuptools==68.0.0
+Shapely==2.0.2
+shortuuid==1.0.11
+slack_sdk==3.26.1
+Sphinx==7.2.6
+sphinx_rtd_theme==2.0.0
+supervision==0.17.1
+tabulate==0.9.0
+termcolor==2.4.0
+torch==2.0.1
+torchvision==0.15.2
+tqdm==4.66.1
+transformers==4.31.0
+typer==0.9.0
+typing_extensions==4.9.0
+uvicorn==0.25.0
+websockets==11.0.3
+Werkzeug==2.1.0
+yapf==0.40.2
+timm
+opencv-python

approach/run_vlm.py

@@ -0,0 +1,164 @@
+"""Configurable screenshot-context mining entrypoint for Orienter."""
+
+import argparse
+import json
+import os
+import sys
+import tempfile
+from pathlib import Path
+
+if __package__ in {None, ""}:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from approach.config import get_model_profile
+from approach.pipeline_utils import (
+    output_path_for_selection,
+    resolve_image_path,
+    select_jsonl_lines,
+    write_json_atomic,
+)
+
+
+def _load_jsonl(path: Path):
+    if not path.exists():
+        return []
+    with path.open(encoding="utf-8") as file:
+        return [json.loads(line) for line in file if line.strip()]
+
+
+def _write_jsonl_atomic(path: Path, records):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    descriptor, temporary = tempfile.mkstemp(
+        prefix=".orienter-",
+        suffix=".jsonl.tmp",
+        dir=path.parent,
+    )
+    try:
+        with os.fdopen(descriptor, "w", encoding="utf-8") as file:
+            for record in records:
+                file.write(json.dumps(record, ensure_ascii=False) + "\n")
+        os.replace(temporary, path)
+    except Exception:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+        raise
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--questions", type=Path, required=True)
+    parser.add_argument("--images-dir", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--profile", default="default")
+    parser.add_argument(
+        "--app-metadata-cache",
+        type=Path,
+        default=(
+            Path(os.environ["ORIENTER_APP_METADATA_CACHE"])
+            if os.environ.get("ORIENTER_APP_METADATA_CACHE")
+            else None
+        ),
+        help="optional JSON/JSONL app metadata cache keyed by app_id; missing app IDs fail closed",
+    )
+    parser.add_argument("--ablation", action="store_true")
+    parser.add_argument("--start-index", type=int)
+    parser.add_argument("--end-index", type=int)
+    parser.add_argument("--shard-index", type=int)
+    parser.add_argument("--num-shards", type=int)
+    parser.add_argument("--resume", action="store_true")
+    return parser
+
+
+def run(args, processor=None):
+    app_metadata_cache = None
+    uses_default_processor = processor is None
+    if processor is None:
+        from approach.vlm.gpt4v.gpt4v import load_app_metadata_cache, process_image
+
+        app_metadata_cache = load_app_metadata_cache(args.app_metadata_cache)
+        processor = process_image
+
+    questions = _load_jsonl(args.questions)
+    selected = select_jsonl_lines(
+        questions,
+        start_index=args.start_index,
+        end_index=args.end_index,
+        shard_index=args.shard_index,
+        num_shards=args.num_shards,
+    )
+    output = Path(
+        output_path_for_selection(
+            str(args.output),
+            start_index=args.start_index,
+            end_index=args.end_index,
+            shard_index=args.shard_index,
+            num_shards=args.num_shards,
+        )
+    )
+    answers = _load_jsonl(output) if args.resume else []
+    completed = {answer["question_id"] for answer in answers}
+    errors = []
+    profile = get_model_profile(args.profile)
+    _write_jsonl_atomic(output, answers)
+
+    for index, question in enumerate(selected):
+        question_id = question["question_id"]
+        if question_id in completed:
+            continue
+        image_path = resolve_image_path(args.images_dir, question["image"])
+        try:
+            processor_args = [
+                args.profile,
+                question.get("text", ""),
+                str(image_path),
+                args.ablation,
+                index,
+            ]
+            if uses_default_processor:
+                result = processor(*processor_args, metadata_cache=app_metadata_cache)
+            else:
+                result = processor(*processor_args)
+            if not result:
+                raise RuntimeError("provider returned an empty response")
+            answers.append(
+                {
+                    "question_id": question_id,
+                    "prompt": question.get("text", ""),
+                    "text": result,
+                    "model_id": profile.model,
+                    "metadata": {"profile": profile.name, "provider": profile.provider},
+                }
+            )
+            completed.add(question_id)
+            _write_jsonl_atomic(output, answers)
+        except Exception as exc:
+            errors.append(
+                {
+                    "question_id": question_id,
+                    "image": question["image"],
+                    "error_type": type(exc).__name__,
+                    "message": str(exc),
+                }
+            )
+
+    error_path = output.with_suffix(".errors.json")
+    write_json_atomic(str(error_path), errors)
+    return {
+        "output": str(output),
+        "records_selected": len(selected),
+        "completed": len(answers),
+        "errors": len(errors),
+        "profile": profile.name,
+        "model": profile.model,
+    }
+
+
+def main(argv=None):
+    args = build_parser().parse_args(argv)
+    report = run(args)
+    print(json.dumps(report, indent=2))
+    return 1 if report["errors"] else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dataset/2detection.py

@@ -0,0 +1,21 @@
+import json
+import argparse
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--input', type=str, required=True)
+parser.add_argument('--output', type=str, required=True)
+args = parser.parse_args()
+
+
+if __name__ == '__main__':
+    with open(args.input, 'r') as f:
+        data = json.load(f)
+
+    for ann in data['annotations']:
+        ann.pop('segmentation')
+        ann['iscrowd'] = 0
+
+    os.makedirs(os.path.dirname(args.output), exist_ok=True)
+    with open(args.output, 'w') as f:
+        json.dump(data, f)

dataset/2polyseg.py

@@ -0,0 +1,52 @@
+import os
+import argparse
+import numpy as np
+import cv2
+import json
+from tqdm import tqdm
+import pycocotools.mask as mask_util
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--input', type=str, required=True)
+parser.add_argument('--output', type=str, required=True)
+args = parser.parse_args()
+
+
+def rle_to_polygon(rle) -> (list, list, float):
+    mask = mask_util.decode(rle)
+    mask = np.where(mask > 0.5, 1, 0)
+    mask = np.ascontiguousarray(mask, dtype=np.uint8)
+    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)
+    polygon = []
+    segs = []
+    contours = [contour for contour in contours if contour.size >= 6]
+    for contour in contours:
+        _contour = contour.flatten().tolist()
+        _contour.append(_contour[0])
+        _contour.append(_contour[1])
+        polygon.extend(_contour)
+        segs.append(contour.flatten().tolist())
+    RLEs = mask_util.frPyObjects(segs, mask.shape[0], mask.shape[1])
+    RLE = mask_util.merge(RLEs)
+    area = float(mask_util.area(RLE))
+    return polygon, area
+
+
+def main(args):
+    with open(args.input, 'r') as f:
+        ann_data = json.load(f)
+    for ann in tqdm(ann_data['annotations'], total=len(ann_data['annotations'])):
+        rle_data = ann['segmentation']
+        # encodedRLE = mask_util.frPyObjects(rle_data, rle_data['size'][0], rle_data['size'][1])
+        # polygon, area = rle_to_polygon(encodedRLE)
+        polygon, area = rle_to_polygon(rle_data)
+        ann['segmentation'] = [polygon]
+        ann['area'] = area
+        ann['iscrowd'] = 0
+    os.makedirs(os.path.dirname(args.output), exist_ok=True)
+    with open(args.output, 'w') as f:
+        json.dump(ann_data, f)
+
+
+if __name__ == '__main__':
+    main(args)

dataset/clean.py

@@ -0,0 +1,55 @@
+# clean up the merged COCO dataset
+from os.path import join as pjoin
+import json
+import pycocotools.mask as mask_util
+
+ROOT = 'data/coco_merged/annotations'
+JSONS = ['interaction.json', 'interactable.json', 'semantics.json']
+
+# filter noise produced by labeling tool, denote mask in compressed RLE format
+def fix_mask(data):
+    for anno in data['annotations']:
+        mask = mask_util.frPyObjects(anno['segmentation'], 540, 960)
+        mask = mask_util.decode(mask)
+        mask[:, -1] = mask[:, -2]
+        mask[-1:, :] = mask[-2, :]
+        RLE = mask_util.encode(mask)
+        bbox = mask_util.toBbox(RLE)
+        bbox = bbox.astype(int)
+        area = int(bbox[2] * bbox[3])
+        RLE['counts'] = RLE['counts'].decode('utf-8')
+        anno['segmentation'] = RLE
+        anno['bbox'] = bbox.tolist()
+        anno['area'] = area
+
+# remove redundant annotations
+def remove_redundant(data):
+    total = len(data['annotations'])
+    anno_list = []
+
+    for anno in data['annotations']:
+        anno['id']=0
+        if anno not in anno_list:
+            anno_list.append(anno)
+    
+    for anno in anno_list:
+        anno['id'] = anno_list.index(anno) + 1
+
+    data['annotations'] = anno_list
+    print(f'{total - len(anno_list)} redundant annotations removed')
+
+
+def main():
+    json_files = [pjoin(ROOT, j) for j in JSONS]
+    for j in json_files:
+        with open(j, 'r') as f:
+            data = json.load(f)
+            fix_mask(data)
+            remove_redundant(data)
+        new_j = j.replace('.json', '_cleaned.json')
+        with open(new_j, 'w') as f:
+            json.dump(data, f, indent=4)
+
+
+if __name__ == '__main__':
+    main()

dataset/create_test_set_folder.py

@@ -0,0 +1,39 @@
+import os
+import shutil
+
+# Define the paths
+image_directory = './data/coco_det/images/semantics'  # The directory containing the images
+txt_file_path = './list_union3.txt'       # The text file with indexes
+new_directory = './data/coco_det/images/union3'      # The directory to copy matching images
+
+# Create the new directory if it doesn't exist
+if not os.path.exists(new_directory):
+    os.makedirs(new_directory)
+
+# Load the comma-separated indexes from the text file
+with open(txt_file_path, 'r') as file:
+    content = file.read().strip()  # Read the entire file content
+    indexes = content.split(', ')   # Split by commas to get a list of indexes
+
+# Iterate over the files in the image directory
+for image_file in os.listdir(image_directory):
+    if image_file.endswith(".jpg"):
+        # Parse the appid and index from the image filename (example: 269170_1.jpg)
+        image_appid, image_index = image_file.split("_")[0], image_file.split("_")[1].split(".")[0].zfill(3)
+        
+        # Combine appid and index to match the format in the text file (example: 269170001)
+        image_key = f"{image_appid}{image_index}"
+        
+        # If the constructed key matches any entry in the text file, copy the image
+        if image_key in indexes:
+            source_path = os.path.join(image_directory, image_file)
+            destination_path = os.path.join(new_directory, image_file)
+            
+            # Copy the image
+            shutil.copy(source_path, destination_path)
+            print(f"Copied: {image_file}")
+
+print("All matching images have been copied.")
+
+
+

dataset/dataset_process.sh

@@ -0,0 +1,32 @@
+# # obtain interactability dataset
+# python interactable_coco.py --input ./data/coco_merged/annotations/semantics.json --output ./data/coco_merged/annotations/interactable.json
+# ls ./data/coco_merged/images/interactable
+# if [ $? -ne 0 ]; then
+#     cp -r ./data/coco_merged/images/semantics ./data/coco_merged/images/interactable
+# fi
+
+# # map interaction dataset from 61 to 54 classes
+# python map_interaction.py
+
+# # generate detection only annotations
+# python 2detection.py --input ./data/coco_merged/annotations/interactable.json --output ./data/coco_det/annotations/interactable.json
+# python 2detection.py --input ./data/coco_merged/annotations/interaction.json --output ./data/coco_det/annotations/interaction.json
+# python 2detection.py --input ./data/coco_merged/annotations/semantics.json --output ./data/coco_det/annotations/semantics.json
+
+# # generate correct segmentation annotations with polygon format
+# python 2polyseg.py --input ./data/coco_merged/annotations/interactable.json --output ./data/coco_seg/annotations/interactable.json
+# python 2polyseg.py --input ./data/coco_merged/annotations/interaction.json --output ./data/coco_seg/annotations/interaction.json
+# python 2polyseg.py --input ./data/coco_merged/annotations/semantics.json --output ./data/coco_seg/annotations/semantics.json
+
+# # split dataset into train
+# export FORMAT=det
+# export TASK=semantics
+# python split_coco.py \
+#     --ann_file ./data/coco_$FORMAT/annotations/$TASK.json \
+#     --img_dir ./data/coco_merged/images/$TASK/ \
+#     --output_path ./data/split_$TASK/ \
+#     --fold_file './fold_app.csv' \
+#     --train_fold '0,1,4,5,7,9' \
+#     --val_fold '3' \
+#     --test_fold '2,6,8'
+# # use --gen to generate fold_file

dataset/fold_app.csv

@@ -0,0 +1,1553 @@
+img_id,fold
+1337530001,0
+2025000010,0
+2025000009,0
+2025000008,0
+2025000007,0
+2025000006,0
+2025000005,0
+2025000004,0
+2025000003,0
+2025000002,0
+2025000001,0
+1960620016,0
+1960620015,0
+1960620014,0
+1960620013,0
+1960620012,0
+1960620011,0
+1960620010,0
+1960620008,0
+1960620002,0
+451980001,0
+451980002,0
+451980003,0
+1150310001,0
+451980018,0
+451980017,0
+1245640021,0
+451980015,0
+451980014,0
+451980013,0
+451980012,0
+451980011,0
+451980010,0
+1960620001,0
+451980009,0
+451980007,0
+451980006,0
+451980005,0
+451980004,0
+1150310002,0
+1150310003,0
+1150310004,0
+1150310005,0
+1150310006,0
+451980008,0
+1601810001,0
+1601810002,0
+1601810003,0
+1245640001,0
+1245640002,0
+1245640003,0
+1245640004,0
+1245640005,0
+1245640006,0
+1245640007,0
+1245640008,0
+1245640009,0
+1245640038,0
+1245640010,0
+1245640012,0
+1245640013,0
+1245640014,0
+1245640015,0
+1245640016,0
+1245640017,0
+1245640018,0
+1245640019,0
+1245640020,0
+1245640011,0
+1337530082,0
+1245640037,0
+1217930003,0
+1612820001,0
+1612820002,0
+1245640036,0
+1245640035,0
+1245640034,0
+1245640033,0
+1245640032,0
+1245640031,0
+1337530002,0
+1217930004,0
+1245640030,0
+1245640028,0
+1245640027,0
+1245640026,0
+1245640025,0
+1245640024,0
+1245640023,0
+1217930005,0
+1217930001,0
+1217930002,0
+1245640029,0
+1337530081,0
+451980016,0
+1337530079,0
+1337530033,0
+1337530034,0
+1337530035,0
+1337530036,0
+1337530037,0
+1337530038,0
+1337530039,0
+1337530040,0
+1337530041,0
+1337530042,0
+1337530043,0
+1337530044,0
+1337530045,0
+1337530046,0
+1337530047,0
+1337530048,0
+1337530049,0
+1337530050,0
+1337530051,0
+1337530052,0
+1337530053,0
+1337530054,0
+1337530055,0
+1337530056,0
+1337530080,0
+1337530032,0
+1337530031,0
+1337530030,0
+1337530029,0
+1337530003,0
+1337530004,0
+1337530005,0
+1337530006,0
+1337530007,0
+1337530008,0
+1337530009,0
+1337530010,0
+1337530011,0
+1337530012,0
+1337530013,0
+1337530014,0
+660520014,0
+1337530015,0
+1337530017,0
+1337530018,0
+1337530019,0
+1337530020,0
+1337530021,0
+1337530022,0
+1337530023,0
+1337530024,0
+1337530025,0
+1337530026,0
+1337530027,0
+1337530028,0
+1337530016,0
+660520013,0
+660520015,0
+1337530058,0
+1337530057,0
+1337530059,0
+1337530060,0
+1337530061,0
+1337530062,0
+1337530063,0
+1337530064,0
+1337530065,0
+1337530066,0
+1337530067,0
+1337530068,0
+1337530069,0
+1337530070,0
+1337530071,0
+1337530072,0
+1337530073,0
+1337530074,0
+1337530075,0
+1337530076,0
+1337530077,0
+1337530078,0
+660520001,0
+660520002,0
+1245640022,0
+660520006,0
+660520008,0
+660520009,0
+660520010,0
+660520005,0
+660520004,0
+660520011,0
+660520003,0
+660520012,0
+660520007,0
+1595490004,1
+1512840006,1
+1512840005,1
+1512840004,1
+1512840003,1
+1512840002,1
+1512840001,1
+1595490014,1
+1512840007,1
+1595490001,1
+1595490017,1
+1595490018,1
+1595490019,1
+1595490020,1
+1295570001,1
+1595490013,1
+1595490012,1
+1595490011,1
+1595490010,1
+1595490009,1
+1595490016,1
+1595490008,1
+1595490007,1
+1595490006,1
+1595490003,1
+1595490005,1
+1595490015,1
+1931980002,1
+1730290004,1
+457380018,1
+457380017,1
+457380016,1
+457380015,1
+457380014,1
+457380013,1
+457380012,1
+457380011,1
+457380010,1
+457380009,1
+812460003,1
+812460004,1
+812460005,1
+812460006,1
+812460007,1
+457380008,1
+457380007,1
+457380006,1
+457380005,1
+457380019,1
+457380004,1
+812460002,1
+716260021,1
+716260006,1
+716260005,1
+716260004,1
+716260003,1
+716260002,1
+716260001,1
+631660001,1
+716260009,1
+716260010,1
+716260011,1
+716260012,1
+716260013,1
+716260014,1
+716260015,1
+716260016,1
+716260017,1
+716260018,1
+716260019,1
+716260020,1
+812460001,1
+457380003,1
+457380002,1
+457380001,1
+1931980009,1
+1931980008,1
+716260007,1
+1931980006,1
+954160002,1
+1931980005,1
+1931980010,1
+1931980004,1
+954160001,1
+1931980001,1
+1730290005,1
+1730290003,1
+1730290002,1
+1730290001,1
+1931980003,1
+1931980011,1
+1931980007,1
+1931980013,1
+812460008,1
+812460009,1
+812460010,1
+1931980012,1
+716260008,1
+1931980014,1
+815280012,2
+998660003,2
+998660002,2
+998660001,2
+815280011,2
+815280010,2
+815280001,2
+815280002,2
+815280003,2
+815280004,2
+815280005,2
+815280006,2
+815280009,2
+815280018,2
+815280017,2
+815280016,2
+815280015,2
+815280014,2
+815280013,2
+815280008,2
+815280007,2
+1652800010,2
+622310002,2
+438100004,2
+438100003,2
+438100002,2
+438100001,2
+2193270001,2
+2020950003,2
+2020950002,2
+2020950001,2
+1652800001,2
+438100005,2
+1652800002,2
+1652800004,2
+1652800005,2
+1652800006,2
+1652800007,2
+1652800008,2
+998660004,2
+1652800013,2
+1652800012,2
+1652800011,2
+1652800003,2
+622310001,2
+438100006,2
+438100008,2
+591680013,2
+591680012,2
+591680011,2
+591680010,2
+591680009,2
+591680008,2
+591680007,2
+591680006,2
+591680005,2
+438100007,2
+591680004,2
+1652800009,2
+591680001,2
+1394410001,2
+1394410002,2
+1394410003,2
+1394410004,2
+1394410005,2
+438100010,2
+438100009,2
+591680003,2
+591680002,2
+998660005,2
+1157070006,2
+1157070001,2
+1157070002,2
+1157070003,2
+1157070008,2
+1157070007,2
+1157070004,2
+1157070005,2
+898080011,3
+898080010,3
+898080009,3
+898080008,3
+898080007,3
+898080006,3
+898080005,3
+898080004,3
+1337060040,3
+898080003,3
+898080002,3
+898080001,3
+1485260001,3
+1485260002,3
+1485260003,3
+1485260004,3
+1485260005,3
+1486660001,3
+1486660002,3
+1486660003,3
+1404360029,3
+1404360027,3
+1404360022,3
+1404360025,3
+898080013,3
+898080012,3
+1404360001,3
+1404360002,3
+1404360003,3
+1404360004,3
+1404360005,3
+1404360006,3
+1404360007,3
+1404360026,3
+1404360009,3
+1404360011,3
+1404360013,3
+1404360015,3
+1404360016,3
+1404360018,3
+1404360019,3
+1486660004,3
+1404360023,3
+1404360024,3
+1404360010,3
+1486660005,3
+1486660010,3
+1486660008,3
+1825460004,3
+1825460003,3
+1825460002,3
+1825460001,3
+528580004,3
+528580003,3
+528580002,3
+528580001,3
+1801560011,3
+1801560010,3
+1801560009,3
+1801560008,3
+1801560007,3
+1801560006,3
+1801560005,3
+1801560004,3
+1801560003,3
+1801560002,3
+1801560001,3
+1825460005,3
+1825460006,3
+1825460007,3
+1825460008,3
+1486660009,3
+898080014,3
+1486660011,3
+1486660012,3
+1486660013,3
+1486660014,3
+1825460012,3
+1825460011,3
+731790010,3
+1486660007,3
+731790009,3
+731790007,3
+731790006,3
+731790005,3
+731790004,3
+731790003,3
+731790002,3
+731790001,3
+1825460010,3
+1825460009,3
+731790008,3
+898080015,3
+1486660006,3
+1394210010,3
+1337060006,3
+1337060005,3
+1337060004,3
+1337060003,3
+1337060002,3
+1337060001,3
+1337060041,3
+1337060007,3
+1337060027,3
+1337060030,3
+1337060031,3
+1337060032,3
+1337060033,3
+898080036,3
+1337060034,3
+1337060035,3
+1337060029,3
+1337060008,3
+1337060009,3
+1337060010,3
+1337060019,3
+1337060020,3
+1337060021,3
+1337060018,3
+1337060017,3
+1337060016,3
+1337060015,3
+1337060022,3
+1337060023,3
+1337060024,3
+898080016,3
+1337060025,3
+1337060026,3
+1337060014,3
+1337060013,3
+1337060012,3
+1337060011,3
+1337060036,3
+1337060039,3
+1337060028,3
+898080029,3
+898080025,3
+898080024,3
+898080023,3
+898080022,3
+898080021,3
+898080020,3
+898080019,3
+898080018,3
+898080028,3
+898080026,3
+898080030,3
+898080031,3
+898080032,3
+898080033,3
+898080017,3
+898080027,3
+1394210002,3
+1394210001,3
+1394210008,3
+1394210009,3
+898080035,3
+1394210006,3
+1394210007,3
+1394210004,3
+1394210003,3
+898080034,3
+1394210005,3
+457550003,4
+1705030007,4
+457550002,4
+457550001,4
+457550004,4
+1705030006,4
+457550005,4
+1705030008,4
+1805510025,4
+1705030010,4
+1805510024,4
+1805510026,4
+1172280004,4
+1172280003,4
+1172280002,4
+1172280001,4
+1172280009,4
+1705030011,4
+1705030012,4
+1705030013,4
+1705030009,4
+1705030001,4
+1805510020,4
+1805510022,4
+1248270013,4
+1248270012,4
+1248270011,4
+1248270010,4
+1248270009,4
+1248270008,4
+1248270007,4
+1248270006,4
+1248270005,4
+1248270004,4
+1248270003,4
+1248270002,4
+1248270001,4
+1705030005,4
+1705030004,4
+1705030003,4
+1705030016,4
+463290001,4
+1805510001,4
+1805510002,4
+1805510003,4
+1805510021,4
+1705030002,4
+1805510019,4
+1805510018,4
+1805510017,4
+1805510016,4
+1805510015,4
+1805510014,4
+1805510023,4
+1805510013,4
+1805510011,4
+1805510010,4
+1805510009,4
+1805510008,4
+1805510007,4
+1805510006,4
+1805510005,4
+1805510004,4
+1805510012,4
+463290002,4
+667010003,4
+463290004,4
+1178780007,4
+1178780008,4
+1178780009,4
+1178780010,4
+1178780011,4
+1178780012,4
+1705030014,4
+463290003,4
+1178780014,4
+910190007,4
+910190006,4
+910190005,4
+910190004,4
+910190003,4
+910190002,4
+910190001,4
+1172280005,4
+1178780016,4
+1178780015,4
+1382350005,4
+1382350004,4
+1382350003,4
+1382350002,4
+1382350001,4
+1382350008,4
+1382350007,4
+1382350006,4
+1178780006,4
+1178780005,4
+1178780013,4
+1705030015,4
+667010001,4
+1805510032,4
+1805510031,4
+667010002,4
+667010004,4
+667010005,4
+667010006,4
+667010007,4
+667010008,4
+667010010,4
+667010011,4
+667010012,4
+1805510033,4
+1172280006,4
+667010009,4
+1172280008,4
+463290005,4
+1172280007,4
+463290007,4
+463290008,4
+1805510027,4
+1805510028,4
+463290006,4
+1805510030,4
+1178780004,4
+1178780003,4
+1178780002,4
+1178780001,4
+1805510029,4
+1707840007,5
+1707840006,5
+1707840005,5
+1707840004,5
+1707840003,5
+2224020003,5
+1707840001,5
+2224020002,5
+2224020001,5
+1707840008,5
+1707840002,5
+2224020030,5
+2224020024,5
+2224020028,5
+2224020023,5
+2224020022,5
+2224020021,5
+2224020020,5
+2224020019,5
+2224020018,5
+2224020017,5
+2224020016,5
+2224020015,5
+2224020014,5
+2224020029,5
+2224020013,5
+2224020011,5
+2224020010,5
+2224020025,5
+2224020008,5
+2224020007,5
+2224020006,5
+2224020005,5
+2224020004,5
+2224020026,5
+2224020027,5
+2224020012,5
+2224020009,5
+2308940007,5
+1695870005,5
+726910008,5
+726910007,5
+726910006,5
+726910005,5
+726910004,5
+726910003,5
+726910002,5
+726910001,5
+726910009,5
+605850001,5
+513490001,5
+497820003,5
+497820002,5
+497820001,5
+2308940001,5
+2308940004,5
+2308940003,5
+2308940002,5
+513490002,5
+726910010,5
+726910011,5
+758210001,5
+1334900005,5
+1334900004,5
+1334900003,5
+1334900002,5
+1334900001,5
+1334900006,5
+1695870006,5
+758210014,5
+758210013,5
+758210012,5
+758210011,5
+758210010,5
+758210009,5
+758210008,5
+758210007,5
+758210005,5
+758210004,5
+758210003,5
+758210002,5
+2308940017,5
+2308940016,5
+758210006,5
+2308940014,5
+2224020045,5
+2224020044,5
+2224020043,5
+2224020042,5
+2224020041,5
+2224020040,5
+2224020039,5
+2224020038,5
+2224020046,5
+2224020037,5
+2224020035,5
+2224020034,5
+1695870004,5
+2308940015,5
+2224020033,5
+1695870003,5
+1695870002,5
+1695870001,5
+2224020032,5
+2224020047,5
+2224020036,5
+2224020049,5
+2308940013,5
+2308940012,5
+2308940011,5
+2308940010,5
+2308940009,5
+2224020048,5
+2308940008,5
+2308940006,5
+2308940005,5
+2224020031,5
+2224020057,5
+2224020056,5
+2224020050,5
+2224020051,5
+2224020052,5
+2224020055,5
+2224020058,5
+2224020054,5
+2224020053,5
+2057000131,6
+2057000130,6
+2057000129,6
+2057000128,6
+2057000126,6
+2057000125,6
+2057000124,6
+2057000123,6
+2057000127,6
+2057000132,6
+2057000134,6
+2057000148,6
+2057000135,6
+2057000136,6
+2057000137,6
+2057000138,6
+2057000140,6
+2057000141,6
+2057000142,6
+2057000143,6
+2057000145,6
+2057000146,6
+2057000147,6
+2057000149,6
+2057000133,6
+2057000139,6
+2057000107,6
+2057000121,6
+2057000093,6
+2057000150,6
+2057000094,6
+2057000095,6
+2057000096,6
+2057000097,6
+2057000098,6
+2057000099,6
+2057000100,6
+2057000101,6
+2057000102,6
+2057000103,6
+2057000104,6
+2057000122,6
+2057000105,6
+2057000108,6
+2057000109,6
+2057000110,6
+2057000111,6
+2057000112,6
+2057000113,6
+2057000114,6
+2057000115,6
+2057000116,6
+2057000117,6
+2057000118,6
+2057000119,6
+2057000120,6
+2057000106,6
+2057000151,6
+997760004,6
+2057000153,6
+997760008,6
+997760007,6
+997760006,6
+997760005,6
+997760003,6
+997760002,6
+997760001,6
+891960010,6
+891960009,6
+2089520003,6
+2089520004,6
+891960008,6
+997760009,6
+891960007,6
+891960005,6
+891960004,6
+891960003,6
+891960002,6
+891960001,6
+790750006,6
+790750005,6
+790750004,6
+790750003,6
+790750002,6
+790750001,6
+2057000092,6
+891960006,6
+997760010,6
+997760011,6
+997760012,6
+2057000154,6
+2057000155,6
+2057000156,6
+2057000157,6
+2057000158,6
+2057000159,6
+2057000160,6
+2057000161,6
+2057000162,6
+2057000163,6
+2057000164,6
+2057000165,6
+2057000166,6
+2057000167,6
+2057000168,6
+2057000169,6
+2057000170,6
+2057000171,6
+2057000172,6
+2057000173,6
+2057000174,6
+2057000175,6
+2057000176,6
+2057000177,6
+2057000178,6
+2089520001,6
+2089520002,6
+2057000152,6
+2057000091,6
+2057000144,6
+2057000089,6
+1269890008,6
+1269890007,6
+1269890006,6
+1269890005,6
+1269890004,6
+1269890003,6
+1269890009,6
+1388030001,6
+1269890001,6
+1456280001,6
+1456280002,6
+1456280003,6
+1456280004,6
+1480650001,6
+1269890002,6
+1269890010,6
+2057000001,6
+2057000002,6
+2057000012,6
+2057000011,6
+2057000010,6
+2057000009,6
+2057000008,6
+2057000007,6
+2057000006,6
+1561560011,6
+2057000004,6
+2057000003,6
+1269890015,6
+1269890014,6
+1269890013,6
+1269890012,6
+1269890011,6
+1480650002,6
+1480650003,6
+1561560032,6
+1561560031,6
+1561560022,6
+1561560021,6
+1561560020,6
+1561560019,6
+1561560018,6
+1561560017,6
+1561560016,6
+1561560015,6
+1561560014,6
+1561560013,6
+1561560006,6
+1561560007,6
+1561560008,6
+1561560009,6
+1561560010,6
+1561560023,6
+2057000013,6
+1561560024,6
+2057000090,6
+1561560030,6
+1561560001,6
+1561560002,6
+1561560003,6
+1480650004,6
+1480650005,6
+1480650006,6
+1480650007,6
+1480650008,6
+1561560004,6
+1561560005,6
+1561560029,6
+1561560028,6
+1561560027,6
+1561560026,6
+1561560025,6
+2057000014,6
+2057000005,6
+2057000016,6
+2057000068,6
+2057000067,6
+2057000066,6
+2057000065,6
+2057000064,6
+2057000063,6
+2057000062,6
+2057000061,6
+2057000060,6
+2057000015,6
+2057000058,6
+2057000057,6
+2057000056,6
+2057000055,6
+2057000054,6
+2057000069,6
+2057000070,6
+2057000071,6
+2057000072,6
+2057000088,6
+2057000087,6
+2057000086,6
+2057000085,6
+2057000084,6
+2057000083,6
+2057000082,6
+2057000053,6
+2057000081,6
+2057000079,6
+2057000078,6
+2057000077,6
+2057000076,6
+2057000075,6
+2057000074,6
+2057000073,6
+2057000080,6
+2057000052,6
+2057000059,6
+2057000050,6
+2057000032,6
+2057000031,6
+2057000030,6
+2057000029,6
+2057000028,6
+2057000051,6
+2057000027,6
+2057000026,6
+2057000025,6
+2057000024,6
+2057000023,6
+2057000022,6
+2057000021,6
+2057000020,6
+2057000019,6
+2057000018,6
+2057000017,6
+2057000033,6
+2057000034,6
+1561560012,6
+2057000036,6
+2057000035,6
+2057000044,6
+2057000045,6
+2057000046,6
+2057000047,6
+2057000043,6
+2057000049,6
+2057000048,6
+2057000042,6
+2057000041,6
+2057000040,6
+2057000039,6
+2057000038,6
+2057000037,6
+1063530027,7
+1063530026,7
+1063530025,7
+1063530028,7
+1063530043,7
+1063530040,7
+1063530041,7
+1550280038,7
+1063530039,7
+1063530038,7
+1063530037,7
+1063530036,7
+1063530035,7
+1063530034,7
+1063530033,7
+1063530032,7
+1063530031,7
+1063530030,7
+1063530042,7
+1063530029,7
+490250005,7
+1063530045,7
+1550280045,7
+1550280046,7
+1550280047,7
+490250019,7
+490250018,7
+490250017,7
+490250015,7
+490250014,7
+490250013,7
+490250012,7
+490250011,7
+490250010,7
+490250009,7
+490250008,7
+490250007,7
+490250006,7
+1063530024,7
+1550280044,7
+1063530044,7
+1550280043,7
+1550280041,7
+1063530046,7
+1063530047,7
+1063530048,7
+1063530049,7
+1063530050,7
+1063530051,7
+1063530052,7
+1063530053,7
+533970003,7
+533970002,7
+533970001,7
+529150004,7
+529150003,7
+529150002,7
+529150001,7
+1550280039,7
+1550280040,7
+1550280042,7
+1063530023,7
+1550280018,7
+1550280036,7
+1550280006,7
+1550280007,7
+1550280008,7
+1550280009,7
+1550280010,7
+1550280011,7
+1550280012,7
+1550280013,7
+1550280014,7
+1550280005,7
+1550280015,7
+1550280017,7
+490250004,7
+1550280019,7
+1550280020,7
+1550280021,7
+1550280022,7
+1550280023,7
+1550280024,7
+1550280025,7
+1550280016,7
+1550280004,7
+1550280003,7
+1550280002,7
+1264160017,7
+1264160016,7
+1264160015,7
+1033960001,7
+1033960002,7
+1033960003,7
+1033960004,7
+1033960005,7
+1033960006,7
+1063530001,7
+1481400001,7
+1481400002,7
+1481400003,7
+1264160014,7
+1264160013,7
+1264160012,7
+1264160011,7
+1264160010,7
+1264160009,7
+1264160008,7
+1550280001,7
+720300018,7
+1550280037,7
+720300017,7
+720300015,7
+1063530013,7
+1063530014,7
+1063530015,7
+1063530016,7
+1063530017,7
+1063530018,7
+1063530019,7
+1063530020,7
+1063530021,7
+1063530012,7
+1063530022,7
+1550280027,7
+1550280028,7
+1550280029,7
+1550280030,7
+1550280031,7
+1550280032,7
+1550280033,7
+1550280034,7
+1550280035,7
+1550280026,7
+1063530011,7
+1063530010,7
+1063530009,7
+720300014,7
+720300013,7
+720300012,7
+720300011,7
+720300010,7
+720300009,7
+720300008,7
+720300007,7
+720300006,7
+720300005,7
+720300004,7
+720300003,7
+720300002,7
+720300001,7
+1063530002,7
+1063530003,7
+1063530004,7
+1063530005,7
+1063530006,7
+1063530007,7
+1063530008,7
+720300016,7
+490250003,7
+490250016,7
+490250001,7
+1063530069,7
+1063530070,7
+1063530071,7
+1063530072,7
+1063530073,7
+1063530074,7
+1063530075,7
+1063530076,7
+1063530077,7
+1063530078,7
+1063530079,7
+1063530080,7
+1063530081,7
+1063530083,7
+490250002,7
+1676620007,7
+1264160007,7
+1264160006,7
+1264160005,7
+1676620006,7
+1676620005,7
+1676620004,7
+1264160004,7
+1264160003,7
+1264160002,7
+1264160001,7
+1676620003,7
+1676620002,7
+1676620001,7
+1063530068,7
+1063530067,7
+1063530082,7
+1063530066,7
+1063530061,7
+1063530062,7
+1063530063,7
+1063530064,7
+1063530065,7
+1063530058,7
+1063530057,7
+1063530056,7
+1063530055,7
+1063530054,7
+1063530059,7
+1550280054,7
+1550280048,7
+1550280049,7
+1550280050,7
+1550280055,7
+1063530060,7
+1550280051,7
+1550280052,7
+1550280053,7
+1026760019,8
+1026760020,8
+1026760021,8
+1026760022,8
+1026760027,8
+1026760024,8
+1026760017,8
+1026760025,8
+1026760016,8
+1026760026,8
+1026760018,8
+1026760023,8
+1453730002,8
+1906880002,8
+1906880001,8
+1862180007,8
+1026760015,8
+1862180006,8
+1862180005,8
+1862180004,8
+1862180003,8
+1862180002,8
+1862180001,8
+1250210001,8
+1250210002,8
+1906880003,8
+1026760014,8
+1026760011,8
+1026760012,8
+2077870001,8
+2077870002,8
+2077870003,8
+2077870004,8
+2077870005,8
+1453730014,8
+1453730013,8
+1453730012,8
+1453730011,8
+1453730010,8
+1453730009,8
+1453730008,8
+1453730007,8
+1453730006,8
+1453730005,8
+1453730004,8
+1453730003,8
+1026760006,8
+1026760005,8
+1026760004,8
+1026760003,8
+1453730001,8
+1026760010,8
+1026760009,8
+1026760008,8
+1026760007,8
+866540001,8
+866540003,8
+866540004,8
+1026760013,8
+866540005,8
+866540007,8
+866540008,8
+866540009,8
+866540010,8
+866540011,8
+866540002,8
+1250210003,8
+1026760002,8
+866540006,8
+1250210004,8
+1026760001,8
+600140014,8
+600140012,8
+600140013,8
+1250210005,8
+269170002,8
+269170003,8
+269170004,8
+269170005,8
+269170006,8
+269170007,8
+269170009,8
+600140015,8
+600140016,8
+600140017,8
+600140018,8
+600140019,8
+600140020,8
+600140021,8
+600140022,8
+600140023,8
+600140024,8
+1113370001,8
+600140011,8
+600140010,8
+269170008,8
+600140009,8
+1113370002,8
+269170001,8
+1113370003,8
+1113370004,8
+600140001,8
+1250210007,8
+1250210008,8
+1250210009,8
+600140002,8
+1250210006,8
+600140003,8
+600140008,8
+600140007,8
+1250210011,8
+1250210010,8
+1250210012,8
+600140006,8
+600140005,8
+600140004,8
+2163140007,9
+2163140008,9
+2163140009,9
+2163140006,9
+2163140010,9
+2163140011,9
+2163140012,9
+518580019,9
+518580020,9
+518580021,9
+518580022,9
+518580023,9
+518580008,9
+518580001,9
+518580002,9
+518580003,9
+518580004,9
+518580005,9
+518580006,9
+518580007,9
+2163140013,9
+2163140016,9
+2163140015,9
+2163140014,9
+2163140005,9
+518580024,9
+2163140004,9
+518580015,9
+2163140002,9
+982710007,9
+982710008,9
+982710009,9
+982710010,9
+982710011,9
+982710012,9
+982710013,9
+982710014,9
+982710015,9
+1298890009,9
+1298890010,9
+982710006,9
+1298890012,9
+1298890014,9
+1298890015,9
+1298890016,9
+1298890017,9
+1298890018,9
+1298890019,9
+1298890020,9
+1298890021,9
+518580011,9
+518580010,9
+518580009,9
+1298890013,9
+2163140003,9
+982710005,9
+982710003,9
+2163140001,9
+1210650002,9
+1210650001,9
+518580018,9
+518580017,9
+518580016,9
+518580014,9
+1270910001,9
+1270910002,9
+1270910003,9
+1270910004,9
+982710004,9
+1270910005,9
+1298890002,9
+1298890003,9
+1298890004,9
+1298890005,9
+1298890006,9
+1298890007,9
+1298890008,9
+518580013,9
+518580012,9
+982710001,9
+982710002,9
+1298890001,9
+2163140017,9
+714100001,9
+2163140019,9
+1210650018,9
+1210650017,9
+1210650016,9
+1210650015,9
+1210650014,9
+343740003,9
+1210650012,9
+1210650011,9
+1210650010,9
+1210650009,9
+1210650008,9
+1210650007,9
+1210650006,9
+1210650005,9
+1210650004,9
+1210650003,9
+2163140058,9
+343740002,9
+343740001,9
+1210650019,9
+1210650020,9
+714100002,9
+714100003,9
+714100004,9
+714100005,9
+714100006,9
+714100007,9
+714100008,9
+1210650028,9
+2163140057,9
+1210650027,9
+1210650025,9
+1210650024,9
+2293180001,9
+2293180002,9
+2293180003,9
+1210650023,9
+1210650022,9
+1210650021,9
+1210650026,9
+2163140018,9
+2163140056,9
+2163140054,9
+2163140032,9
+2163140031,9
+2163140030,9
+2163140029,9
+2163140028,9
+2163140027,9
+2163140026,9
+2163140025,9
+2163140024,9
+2163140023,9
+2163140022,9
+2163140021,9
+2163140020,9
+343740004,9
+343740005,9
+343740006,9
+343740007,9
+2163140033,9
+2163140034,9
+1488730001,9
+2163140035,9
+2163140053,9
+2163140052,9
+2163140051,9
+2163140050,9
+2163140049,9
+2163140048,9
+2163140047,9
+2163140046,9
+2163140055,9
+2163140045,9
+2163140043,9
+2163140042,9
+2163140041,9
+2163140040,9
+2163140039,9
+2163140038,9
+2163140037,9
+2163140036,9
+2163140044,9
+1210650013,9

dataset/fold_cat.csv

@@ -0,0 +1,1553 @@
+img_id,fold
+1210650020,0
+866540008,0
+1453730013,0
+343740001,0
+1550280034,0
+2057000087,0
+667010003,0
+1150310003,0
+1270910005,0
+1394210007,0
+2057000101,0
+1337060005,0
+1063530027,0
+600140004,0
+1063530050,0
+1337530072,0
+1550280019,0
+2057000175,0
+1298890018,0
+1705030005,0
+2057000158,0
+1245640028,0
+2025000002,0
+457380004,0
+518580020,0
+2057000141,0
+1063530025,0
+1561560017,0
+490250002,0
+997760003,0
+2057000165,0
+1172280005,0
+1801560005,0
+790750006,0
+812460002,0
+1337060017,0
+982710011,0
+1337530025,0
+1595490014,0
+1245640023,0
+2293180003,0
+2057000146,0
+1063530013,0
+898080033,0
+1337530054,0
+1245640031,0
+2057000066,0
+815280010,0
+2057000168,0
+982710013,0
+490250001,0
+1486660008,0
+1178780004,0
+600140016,0
+1026760002,0
+1113370003,0
+910190006,0
+2025000001,0
+898080014,0
+1595490011,0
+1825460005,0
+1550280040,0
+2163140037,0
+1337060013,0
+2057000070,0
+1705030004,0
+457380002,0
+2057000100,0
+898080011,0
+533970002,0
+1210650022,0
+2057000156,0
+2057000014,0
+891960004,0
+898080012,0
+2057000036,0
+716260007,0
+1063530024,0
+1245640012,0
+1026760003,0
+2057000140,0
+898080008,0
+866540002,0
+1026760014,0
+1269890008,0
+2057000113,0
+1337530082,0
+2057000019,0
+1801560002,0
+1337530073,0
+2163140019,0
+1550280047,0
+2057000149,0
+1801560007,0
+866540003,0
+1270910004,0
+591680003,0
+1270910001,0
+1178780003,0
+1150310006,0
+1550280011,0
+2224020045,0
+2057000060,0
+997760008,0
+1550280003,0
+463290001,0
+1248270006,0
+451980012,0
+1906880002,0
+1931980009,0
+1480650002,0
+1172280007,0
+726910011,0
+998660004,0
+1453730001,0
+815280014,0
+898080006,0
+457550001,0
+490250012,0
+591680007,0
+2057000005,0
+343740007,0
+2057000003,0
+1512840005,0
+997760002,0
+1337530040,0
+457380009,0
+1456280004,0
+1382350003,0
+2057000093,0
+1248270003,0
+2057000151,0
+1705030009,0
+1269890007,0
+1337530066,0
+1063530043,0
+1486660006,0
+898080021,0
+1595490015,0
+2057000067,0
+716260006,0
+660520010,0
+591680001,0
+731790004,0
+1178780008,0
+1245640027,0
+716260018,0
+1337060032,0
+2057000096,0
+1245640033,0
+1862180003,0
+898080032,0
+2163140023,0
+1805510019,0
+1550280048,0
+910190002,0
+891960002,0
+726910008,0
+2057000012,0
+451980007,0
+1481400002,0
+2224020058,0
+1337530060,0
+1561560019,0
+1488730001,0
+1595490016,0
+1269890014,0
+1337530044,0
+1931980014,0
+2308940007,0
+1805510014,0
+1404360015,0
+716260008,0
+2057000057,0
+1337060036,0
+1707840003,0
+2057000129,0
+1550280015,0
+2025000003,0
+518580008,0
+2163140020,0
+1550280002,0
+716260020,0
+1801560009,0
+1862180006,0
+1652800006,0
+1695870003,0
+2163140002,0
+1931980006,0
+2057000132,0
+2057000091,0
+891960001,0
+1801560003,0
+1172280006,0
+660520015,0
+898080002,0
+1026760022,0
+667010007,0
+438100006,0
+457380018,0
+343740004,0
+1245640007,0
+1337530002,0
+1652800003,0
+528580001,0
+1245640021,0
+815280007,0
+1480650007,0
+2057000170,0
+1063530080,0
+2163140010,0
+714100008,0
+2057000038,0
+1453730011,0
+600140001,0
+720300005,0
+605850001,0
+1550280044,0
+269170004,0
+1550280046,0
+1805510031,0
+2057000157,0
+1707840005,0
+726910009,0
+910190005,0
+1561560009,0
+451980017,0
+2163140008,0
+1561560030,0
+1595490013,0
+720300009,0
+891960005,0
+997760001,0
+2057000172,0
+1931980012,0
+2057000125,0
+490250019,0
+2057000085,0
+1705030016,0
+1730290001,0
+529150001,0
+518580009,0
+1157070005,0
+2057000016,0
+1453730014,0
+2224020001,0
+1063530034,0
+2057000090,0
+1250210001,0
+269170008,0
+790750004,0
+1705030015,0
+1150310002,0
+1337530004,0
+982710004,0
+451980010,0
+1337530030,0
+2163140045,0
+1248270002,0
+343740005,0
+2057000164,0
+457380001,0
+2057000139,0
+660520013,0
+1394410004,0
+1805510022,0
+982710015,0
+438100010,0
+1250210009,0
+731790007,0
+660520001,0
+1337060041,0
+716260019,0
+1337530012,0
+716260011,0
+898080034,0
+1561560020,0
+1337060039,0
+1269890015,0
+2308940008,0
+1157070006,0
+1245640017,0
+2163140052,0
+1298890005,0
+457380005,0
+463290006,0
+518580012,0
+1707840002,0
+2057000072,0
+720300014,0
+2057000116,0
+1298890007,0
+463290007,0
+451980018,0
+1245640018,0
+2077870005,0
+2057000044,0
+1337530071,0
+982710010,0
+1337060006,0
+1801560011,0
+1825460008,0
+1334900001,0
+2057000073,0
+1063530037,0
+866540005,0
+1805510013,0
+2163140056,0
+1931980008,0
+1026760009,0
+2057000105,0
+898080013,0
+1453730007,0
+2057000144,0
+1178780013,0
+1063530028,0
+2057000011,0
+1063530051,0
+2057000098,0
+1178780007,0
+2308940004,0
+1150310001,0
+1595490005,0
+2057000025,0
+2057000114,0
+1245640001,0
+2163140036,0
+1453730012,0
+1063530082,0
+2308940006,0
+1862180004,0
+2057000147,0
+1210650009,0
+490250011,0
+1705030012,0
+1550280050,0
+1264160014,0
+1931980013,0
+2057000034,0
+2089520002,0
+1337530026,0
+2163140001,0
+731790003,0
+2057000099,0
+716260014,0
+1026760021,0
+2077870004,0
+667010006,0
+1601810002,0
+1550280033,0
+1561560008,0
+2163140003,0
+1063530039,0
+982710003,0
+1595490012,0
+1480650001,0
+1801560004,0
+898080020,0
+1026760006,0
+1113370001,0
+2293180002,0
+1063530064,0
+1264160016,0
+1652800008,0
+1960620001,0
+790750001,0
+1486660004,0
+2020950002,0
+1026760018,0
+1550280006,0
+1178780014,0
+2057000109,0
+2057000137,0
+898080007,0
+1172280001,0
+667010008,0
+2293180001,0
+2163140032,0
+997760005,0
+660520004,0
+1250210010,0
+2025000008,0
+982710007,0
+1404360011,0
+1334900002,0
+1805510029,0
+1178780006,0
+1805510005,0
+1512840007,0
+1172280003,0
+1250210007,0
+1404360026,0
+812460001,0
+1157070002,0
+1595490009,0
+1337060003,0
+1595490003,0
+438100005,0
+1862180001,0
+1245640019,0
+1404360003,0
+1210650019,0
+1707840006,0
+1248270001,0
+1550280017,0
+898080009,0
+2057000022,0
+2057000035,0
+2308940009,0
+1113370004,0
+1245640013,0
+1486660003,0
+1337530032,0
+726910001,0
+667010002,0
+451980016,0
+2089520001,0
+2057000076,0
+2163140041,0
+1550280024,0
+2163140035,0
+2163140005,0
+451980004,0
+463290005,0
+2057000161,0
+1595490017,0
+998660001,0
+457380019,0
+1337530043,0
+1595490018,0
+2057000136,0
+2057000167,0
+1210650018,0
+1825460007,0
+982710002,0
+1561560027,0
+1652800001,0
+2057000148,0
+529150002,0
+1960620016,0
+1063530078,0
+667010011,0
+1960620010,0
+1337530019,0
+1404360025,0
+1210650021,0
+667010012,0
+600140011,0
+731790001,0
+490250009,0
+2057000020,0
+815280004,0
+2057000145,0
+1063530047,0
+1453730002,0
+1705030010,0
+1337530007,0
+2163140044,0
+1805510028,0
+2057000102,0
+910190003,0
+1394210003,0
+2057000043,0
+2057000108,0
+529150003,0
+451980008,0
+2057000150,0
+1456280001,0
+1595490010,0
+2057000041,0
+716260021,0
+2163140051,0
+1453730005,0
+591680002,0
+1486660001,0
+451980014,0
+518580007,0
+1652800010,0
+1178780001,0
+2057000084,0
+2057000021,0
+1210650007,0
+997760012,0
+1931980005,0
+1337530062,0
+982710012,0
+1550280008,0
+1063530041,0
+1486660013,0
+1394210005,0
+720300007,0
+1210650017,0
+1337530076,0
+1269890005,0
+600140003,0
+1245640020,0
+1033960004,0
+1157070003,0
+815280018,0
+1245640029,0
+1033960002,0
+1210650010,0
+528580002,0
+1269890009,0
+1269890013,0
+982710009,0
+457550005,0
+815280003,0
+1033960006,0
+898080004,0
+457380012,0
+716260009,0
+1157070004,0
+1550280023,0
+451980011,0
+716260010,0
+2057000163,0
+1456280002,0
+1337530064,0
+457380006,0
+716260001,0
+2057000032,0
+1033960005,0
+2057000071,0
+1705030013,0
+451980001,0
+1063530081,0
+898080035,0
+2057000055,0
+2057000024,0
+1337530055,0
+1063530076,0
+269170003,0
+1561560023,0
+1210650008,0
+1394210009,0
+2224020031,0
+1250210002,0
+1801560001,0
+622310002,0
+1026760004,0
+1245640002,0
+1245640035,0
+898080010,0
+1178780005,0
+1805510027,0
+2057000039,0
+1561560018,0
+1825460012,0
+2077870002,0
+1337530065,0
+2057000008,0
+1805510023,0
+891960007,0
+2057000112,0
+1394210006,0
+2057000069,0
+1337530042,0
+667010010,0
+997760010,0
+1805510033,0
+1456280003,0
+2057000154,0
+2163140009,0
+490250006,0
+1394210004,0
+2057000131,0
+1337530001,0
+1801560010,0
+1805510007,0
+726910006,0
+1063530060,0
+457380007,0
+1063530046,0
+1269890004,0
+1264160017,0
+2163140021,0
+2163140054,0
+529150004,0
+898080016,0
+815280002,0
+1063530055,0
+1245640008,0
+457380014,0
+2089520004,0
+660520007,0
+1063530038,0
+1404360013,0
+790750005,0
+1337530016,0
+2057000045,0
+2057000176,0
+1512840004,0
+731790008,0
+1805510004,0
+1026760001,0
+490250010,0
+1676620001,0
+1269890006,0
+1217930001,0
+866540011,0
+1298890002,0
+720300004,0
+518580017,0
+1026760012,0
+1250210005,0
+1337530036,0
+1337530031,0
+1486660014,0
+1337060034,0
+2057000092,0
+866540007,0
+720300013,0
+1026760015,0
+2308940011,0
+600140015,0
+1561560024,0
+1485260003,0
+898080015,0
+518580014,0
+1612820001,0
+2057000110,0
+1931980001,0
+1601810003,0
+2057000130,0
+1695870001,0
+720300010,0
+998660005,0
+2163140047,0
+2057000160,0
+1486660011,0
+2163140031,0
+667010001,0
+1337060040,0
+1595490020,0
+2057000018,0
+1595490007,0
+1245640003,0
+1033960003,0
+1825460004,0
+1337530059,0
+866540004,0
+1337530038,0
+1394210008,0
+1063530030,0
+2057000089,0
+518580022,0
+1825460001,0
+1550280020,0
+1337530035,0
+1337060028,0
+2057000080,0
+1245640004,0
+1705030006,0
+1485260004,0
+1157070007,0
+2057000051,0
+1250210008,0
+2057000115,0
+1063530031,0
+1264160001,0
+343740006,0
+731790005,0
+660520006,0
+2057000111,0
+1063530068,0
+1512840001,0
+2224020002,0
+2057000128,0
+2057000037,0
+898080017,0
+622310001,0
+2057000103,0
+1404360019,0
+1178780011,0
+1337060029,0
+2163140026,0
+1730290002,0
+1264160015,0
+898080024,0
+660520014,0
+1486660005,0
+2163140013,0
+1550280032,0
+2193270001,0
+2163140018,0
+2057000068,0
+2163140049,0
+1486660009,0
+490250008,0
+716260013,0
+1337530051,0
+1550280016,0
+2163140027,0
+720300016,0
+1705030008,0
+2057000106,0
+1210650006,0
+1337530081,0
+1730290004,0
+1337530033,0
+1485260002,0
+457380015,0
+2057000054,0
+1652800012,0
+2025000007,0
+2057000077,0
+1612820002,0
+891960008,0
+1063530040,0
+1652800009,0
+1862180007,0
+1063530071,0
+600140006,0
+1063530035,0
+1245640024,0
+457380016,0
+2057000026,0
+1960620015,0
+451980002,0
+1695870006,0
+1264160002,0
+660520011,0
+463290004,0
+1063530077,0
+2057000017,0
+1404360001,0
+1862180002,0
+631660001,0
+1250210003,0
+1026760025,0
+898080003,0
+2057000153,0
+1063530056,0
+997760011,0
+1178780012,0
+1550280028,0
+898080029,0
+1298890003,0
+1337530029,0
+1960620012,0
+667010009,0
+518580021,0
+731790010,0
+1931980002,0
+1695870005,0
+2308940014,0
+497820002,0
+1825460006,0
+2057000171,0
+997760007,0
+910190007,0
+2057000095,0
+1382350005,0
+1217930004,0
+1026760005,0
+2057000002,0
+2308940002,0
+1337530047,0
+1217930003,0
+2308940010,0
+2163140030,0
+1337530039,0
+982710014,0
+513490001,0
+910190004,0
+998660002,0
+451980005,0
+997760009,0
+1210650015,0
+451980003,0
+2057000097,0
+898080025,0
+982710001,0
+1931980007,0
+716260012,0
+2089520003,0
+1825460011,0
+1178780016,0
+2308940012,0
+898080036,0
+2057000107,0
+497820001,0
+457380003,0
+2057000050,0
+1113370002,0
+1063530045,0
+343740003,0
+1063530049,0
+1178780010,0
+1269890010,0
+1404360024,0
+2057000031,0
+269170006,0
+1550280007,0
+1063530072,0
+1652800011,0
+2057000033,0
+1707840001,0
+2308940003,0
+910190001,0
+2057000015,0
+1960620008,0
+2057000173,0
+2057000166,0
+1805510032,0
+2057000086,0
+1595490006,0
+1178780009,0
+714100003,0
+1481400001,0
+1337530052,0
+891960003,0
+1707840004,0
+997760004,0
+1210650004,0
+815280016,0
+451980006,0
+1550280009,0
+1960620013,0
+1485260001,0
+714100002,0
+660520002,0
+1150310005,0
+1337530011,0
+1705030014,0
+2057000048,0
+1550280021,0
+891960006,0
+731790002,0
+716260017,0
+1705030001,0
+1063530053,0
+1652800013,0
+720300006,0
+815280001,0
+269170002,0
+866540009,0
+2224020003,0
+490250013,0
+1382350001,0
+1512840002,0
+716260015,0
+600140014,0
+1210650003,0
+1805510020,0
+2057000065,0
+1248270007,0
+438100009,0
+490250018,0
+1337530074,0
+2057000075,0
+1157070008,0
+2163140017,0
+1337530053,0
+1931980004,0
+1453730008,0
+1801560008,0
+1595490004,0
+1063530061,0
+1337060027,0
+2057000056,0
+1248270009,0
+1825460010,0
+1063530063,0
+1805510024,0
+1404360027,0
+1550280026,0
+1063530005,0
+2308940001,0
+269170005,0
+1394210010,0
+1705030002,0
+1245640022,0
+1550280055,0
+891960009,0
+1652800007,0
+438100002,0
+2057000104,0
+1931980003,0
+1486660002,0
+269170007,0
+1337530021,0
+1337530020,0
+1561560029,0
+1033960001,0
+2057000023,0
+1269890011,0
+1245640011,0
+1906880001,0
+2020950001,0
+1601810001,0
+457380011,0
+1394210001,0
+2308940016,0
+1486660007,0
+438100001,0
+1550280014,0
+1550280025,0
+1550280004,0
+1705030007,0
+997760006,0
+720300018,0
+1026760024,0
+866540006,0
+898080005,0
+1805510030,0
+1063530003,0
+1298890001,0
+1210650001,0
+2057000052,0
+1250210011,0
+1394410003,0
+1906880003,0
+1250210006,0
+2057000094,0
+1337530080,0
+1561560022,0
+1248270013,0
+2057000042,0
+1217930005,0
+1298890017,0
+815280009,0
+2057000061,0
+720300011,0
+714100001,0
+1486660010,0
+1337530028,0
+898080023,0
+2057000081,0
+1270910003,0
+2308940017,0
+1337530008,0
+2057000047,0
+2057000135,0
+1245640038,0
+1561560026,0
+1337060002,0
+731790009,0
+1026760016,0
+1264160013,0
+716260004,0
+2057000079,0
+1337530058,0
+1561560004,0
+1801560006,0
+1404360002,0
+1652800004,1
+1652800005,1
+1707840008,1
+591680008,1
+1394410001,1
+1394410002,1
+1394410005,1
+1063530006,1
+1063530017,1
+438100004,1
+1063530020,1
+1063530022,1
+438100007,1
+438100008,1
+1063530026,1
+1063530033,1
+1270910002,1
+1695870002,1
+1695870004,1
+1063530036,1
+1063530042,1
+1063530044,1
+2057000001,1
+660520003,1
+2057000006,1
+1063530054,1
+660520008,1
+2057000010,1
+1063530058,1
+660520012,1
+1063530062,1
+1295570001,1
+2224020049,1
+533970001,1
+533970003,1
+1063530066,1
+513490002,1
+1063530075,1
+343740002,1
+2057000040,1
+2057000046,1
+2057000049,1
+1150310004,1
+2057000058,1
+2057000078,1
+2057000082,1
+1337530015,1
+1337060001,1
+1337060004,1
+1862180005,1
+1337060014,1
+716260016,1
+457380017,1
+1337060016,1
+1157070001,1
+731790006,1
+1337530045,1
+1337060031,1
+1337060033,1
+1172280002,1
+1481400003,1
+1172280004,1
+1172280008,1
+1172280009,1
+891960010,1
+1337530056,1
+1595490001,1
+1394210002,1
+1453730003,1
+1250210004,1
+1337530068,1
+1453730006,1
+2057000152,1
+2057000155,1
+1250210012,1
+1337530078,1
+2057000159,1
+1676620002,1
+1595490019,1
+2308940005,1
+1485260005,1
+451980009,1
+2057000169,1
+1931980010,1
+2308940013,1
+451980013,1
+2308940015,1
+451980015,1
+2057000174,1
+2057000177,1
+600140019,1
+898080001,1
+1264160012,1
+1217930002,1
+490250007,1
+898080030,1
+1298890015,1
+898080031,1
+490250016,1
+714100005,1
+518580013,1
+790750002,1
+790750003,1
+518580018,1
+518580019,1
+1512840003,1
+1550280005,1
+1512840006,1
+1026760007,1
+1026760008,1
+1026760010,1
+1245640010,1
+1026760013,1
+1730290003,1
+1730290005,1
+1550280022,1
+1245640025,1
+1245640026,1
+1026760026,1
+1026760027,1
+1245640030,1
+1178780002,1
+497820003,1
+1550280038,1
+1705030003,1
+1550280054,1
+1705030011,1
+1805510017,1
+1805510018,1
+815280005,1
+1805510025,1
+815280011,1
+815280013,1
+815280015,1
+815280017,1
+1210650016,1
+998660003,1
+2163140028,1
+1561560003,1
+2163140040,1
+2163140046,1
+1269890001,1
+667010004,1
+1561560021,1
+667010005,1
+1561560025,1
+1269890012,1
+1561560028,1
+1561560032,1
+866540001,1
+1960620002,1
+720300001,1
+720300002,1
+866540010,1
+1960620011,1
+1960620014,1
+720300015,1
+2020950003,1
+438100003,2
+1388030001,2
+457380008,2
+457380010,2
+457380013,2
+457550002,2
+457550003,2
+457550004,2
+1453730004,2
+1453730009,2
+1453730010,2
+600140002,2
+1676620003,2
+1676620004,2
+1676620005,2
+1676620006,2
+1676620007,2
+600140005,2
+600140007,2
+600140008,2
+600140009,2
+600140010,2
+600140012,2
+600140013,2
+600140017,2
+600140018,2
+600140020,2
+600140021,2
+600140022,2
+600140023,2
+600140024,2
+1178780015,2
+1486660012,2
+1652800002,2
+591680004,2
+591680005,2
+591680006,2
+1707840007,2
+591680009,2
+591680010,2
+591680011,2
+591680012,2
+591680013,2
+2224020004,2
+2224020005,2
+2224020006,2
+2224020007,2
+2224020008,2
+2224020009,2
+2224020010,2
+2224020011,2
+2224020012,2
+2224020013,2
+2224020014,2
+2224020015,2
+2224020016,2
+2224020017,2
+2224020018,2
+2224020019,2
+2224020020,2
+2224020021,2
+2224020022,2
+2224020023,2
+2224020024,2
+2224020025,2
+2224020026,2
+2224020027,2
+2224020028,2
+2224020029,2
+2224020030,2
+2224020032,2
+2224020033,2
+2224020034,2
+2224020035,2
+2224020036,2
+2224020037,2
+2224020038,2
+2224020039,2
+2224020040,2
+2224020041,2
+2224020042,2
+2224020043,2
+2224020044,2
+2224020046,2
+2224020047,2
+2224020048,2
+2224020050,2
+2224020051,2
+2224020052,2
+2224020053,2
+2224020054,2
+2224020055,2
+2224020056,2
+2224020057,2
+716260002,2
+716260003,2
+716260005,2
+1337060007,2
+1337060008,2
+1337060009,2
+1337060010,2
+1337060011,2
+1337060012,2
+1337060015,2
+1337060018,2
+1337060019,2
+1337060020,2
+1337060021,2
+1337060022,2
+1337060023,2
+1337060024,2
+1337060025,2
+1337060026,2
+1337060030,2
+1337060035,2
+954160001,2
+954160002,2
+1561560001,2
+1561560002,2
+1561560005,2
+1561560006,2
+1561560007,2
+1561560010,2
+1561560011,2
+1561560012,2
+1561560013,2
+1561560014,2
+1561560015,2
+1561560016,2
+1269890002,2
+1269890003,2
+1561560031,2
+726910002,2
+726910003,2
+726910004,2
+726910005,2
+726910007,2
+726910010,2
+2025000004,2
+2025000005,2
+2025000006,2
+2057000004,2
+2057000007,2
+2025000009,2
+2025000010,2
+2057000009,2
+660520005,2
+2057000013,2
+660520009,2
+2057000027,2
+2057000028,2
+2057000029,2
+2057000030,2
+2057000053,2
+2057000059,2
+2057000062,2
+2057000063,2
+2057000064,2
+2057000074,2
+2057000083,2
+2057000088,2
+1382350002,2
+1382350004,2
+2057000117,2
+2057000118,2
+2057000119,2
+2057000120,2
+2057000121,2
+2057000122,2
+2057000123,2
+2057000124,2
+1382350006,2
+2057000126,2
+2057000127,2
+1382350007,2
+1382350008,2
+2057000133,2
+2057000134,2
+2057000138,2
+2057000142,2
+2057000143,2
+2057000162,2
+1931980011,2
+2057000178,2
+1264160003,2
+1264160004,2
+1264160005,2
+1264160006,2
+1264160007,2
+1264160008,2
+1264160009,2
+1264160010,2
+1264160011,2
+898080018,2
+1480650003,2
+1298890004,2
+1480650004,2
+1298890006,2
+1480650005,2
+1298890008,2
+1298890009,2
+1298890010,2
+1480650006,2
+1298890012,2
+1298890013,2
+1298890014,2
+1480650008,2
+1298890016,2
+898080026,2
+898080027,2
+1298890019,2
+1298890020,2
+1298890021,2
+1334900006,2
+1334900003,2
+1334900004,2
+1334900005,2
+714100004,2
+714100006,2
+714100007,2
+1550280001,2
+1404360004,2
+1245640005,2
+1245640006,2
+1404360005,2
+1404360006,2
+1245640009,2
+1404360007,2
+1026760011,2
+1404360009,2
+1404360010,2
+1245640014,2
+1245640015,2
+1245640016,2
+1026760017,2
+1404360016,2
+1026760019,2
+1026760020,2
+1404360018,2
+1404360022,2
+1026760023,2
+1404360023,2
+1550280018,2
+269170001,2
+1550280027,2
+269170009,2
+1404360029,2
+1550280029,2
+1550280030,2
+1245640032,2
+1550280031,2
+1245640034,2
+1550280035,2
+1245640036,2
+1245640037,2
+1550280036,2
+1550280037,2
+1550280039,2
+1550280041,2
+1550280042,2
+1550280043,2
+1550280045,2
+1550280049,2
+1805510001,2
+1550280051,2
+1550280052,2
+1550280053,2
+1805510002,2
+1805510003,2
+1805510006,2
+1805510008,2
+1805510009,2
+1805510010,2
+1805510011,2
+1805510012,2
+1805510015,2
+1805510016,2
+1805510021,2
+1805510026,2
+1210650002,2
+1210650005,2
+1210650011,2
+1210650012,2
+1210650013,2
+1210650014,2
+1550280010,2
+528580003,2
+2163140004,2
+528580004,2
+2163140006,2
+1210650023,2
+1210650024,2
+1210650025,2
+1210650026,2
+1210650027,2
+1210650028,2
+1550280012,2
+1550280013,2
+2163140007,2
+2163140011,2
+2163140012,2
+2163140014,2
+2163140015,2
+2163140016,2
+2163140022,2
+2163140024,2
+2163140025,2
+2163140029,2
+2163140033,2
+2163140034,2
+2163140038,2
+2163140039,2
+2163140042,2
+2163140043,2
+2163140048,2
+758210001,2
+2163140050,2
+758210002,2
+758210003,2
+2163140053,2
+758210004,2
+2163140055,2
+758210005,2
+2163140057,2
+2163140058,2
+758210006,2
+758210007,2
+758210008,2
+758210009,2
+758210010,2
+758210011,2
+758210012,2
+758210013,2
+758210014,2
+1063530001,2
+1063530002,2
+1063530004,2
+1063530007,2
+1063530008,2
+1063530009,2
+1063530010,2
+1063530011,2
+1063530012,2
+1063530014,2
+1063530015,2
+1063530016,2
+1063530018,2
+1063530019,2
+1063530021,2
+1063530023,2
+1063530029,2
+1063530032,2
+1063530048,2
+1063530052,2
+898080019,2
+1063530057,2
+1063530059,2
+1063530065,2
+1063530067,2
+1063530069,2
+1063530070,2
+898080022,2
+1063530073,2
+1063530074,2
+1063530079,2
+1063530083,2
+898080028,2
+463290002,2
+1337530003,2
+463290003,2
+1337530005,2
+1337530006,2
+463290008,2
+1337530009,2
+1337530010,2
+1337530013,2
+1337530014,2
+1337530017,2
+1337530018,2
+1337530022,2
+1337530023,2
+1337530024,2
+1337530027,2
+1337530034,2
+1248270004,2
+1337530037,2
+1248270005,2
+1248270008,2
+1337530041,2
+1248270010,2
+1248270011,2
+1248270012,2
+1337530046,2
+1337530048,2
+1337530049,2
+1337530050,2
+1337530057,2
+1337530061,2
+1337530063,2
+1337530067,2
+1337530069,2
+1337530070,2
+1595490008,2
+1337530075,2
+1337530077,2
+1337530079,2
+982710005,2
+982710006,2
+982710008,2
+490250003,2
+490250004,2
+490250005,2
+490250014,2
+490250015,2
+490250017,2
+1825460002,2
+1825460003,2
+518580001,2
+518580002,2
+518580003,2
+518580004,2
+518580005,2
+1825460009,2
+518580010,2
+518580011,2
+518580006,2
+518580015,2
+518580016,2
+518580023,2
+518580024,2
+815280006,2
+815280008,2
+815280012,2
+2077870001,2
+2077870003,2
+720300003,2
+812460003,2
+812460004,2
+812460005,2
+812460006,2
+720300008,2
+812460007,2
+812460008,2
+812460009,2
+720300012,2
+812460010,2
+720300017,2

dataset/fold_genre.csv

@@ -0,0 +1,1553 @@
+img_id,fold
+343740001,0
+343740002,0
+343740003,0
+343740004,0
+343740005,0
+343740006,0
+343740007,0
+457380001,0
+457380002,0
+457380003,0
+457380004,0
+457380005,0
+457380006,0
+457380007,0
+457380008,0
+457380009,0
+457380010,0
+457380011,0
+457380012,0
+457380013,0
+457380014,0
+457380015,0
+457380016,0
+457380017,0
+457380018,0
+457380019,0
+463290001,0
+463290002,0
+463290003,0
+463290004,0
+463290005,0
+463290006,0
+463290007,0
+463290008,0
+513490001,0
+513490002,0
+518580001,0
+518580002,0
+518580003,0
+518580004,0
+518580005,0
+518580006,0
+518580007,0
+518580008,0
+518580009,0
+518580010,0
+518580011,0
+518580012,0
+518580013,0
+518580014,0
+518580015,0
+518580016,0
+518580017,0
+518580018,0
+518580019,0
+518580020,0
+518580021,0
+518580022,0
+518580023,0
+518580024,0
+528580001,0
+528580002,0
+528580003,0
+528580004,0
+605850001,0
+622310001,0
+622310002,0
+660520001,0
+660520002,0
+660520003,0
+660520004,0
+660520005,0
+660520006,0
+660520007,0
+660520008,0
+660520009,0
+660520010,0
+660520011,0
+660520012,0
+660520013,0
+660520014,0
+660520015,0
+667010001,0
+667010002,0
+667010003,0
+667010004,0
+667010005,0
+667010006,0
+667010007,0
+667010008,0
+667010009,0
+667010010,0
+667010011,0
+667010012,0
+716260001,0
+716260002,0
+716260003,0
+716260004,0
+716260005,0
+716260006,0
+716260007,0
+716260008,0
+716260009,0
+716260010,0
+716260011,0
+716260012,0
+716260013,0
+716260014,0
+716260015,0
+716260016,0
+716260017,0
+716260018,0
+716260019,0
+716260020,0
+716260021,0
+720300001,0
+720300002,0
+720300003,0
+720300004,0
+720300005,0
+720300006,0
+720300007,0
+720300008,0
+720300009,0
+720300010,0
+720300011,0
+720300012,0
+720300013,0
+720300014,0
+720300015,0
+720300016,0
+720300017,0
+720300018,0
+726910001,0
+726910002,0
+726910003,0
+726910004,0
+726910005,0
+726910006,0
+726910007,0
+726910008,0
+726910009,0
+726910010,0
+726910011,0
+790750001,0
+790750002,0
+790750003,0
+790750004,0
+790750005,0
+790750006,0
+812460001,0
+812460002,0
+812460003,0
+812460004,0
+812460005,0
+812460006,0
+812460007,0
+812460008,0
+812460009,0
+812460010,0
+815280001,0
+815280002,0
+815280003,0
+815280004,0
+815280005,0
+815280006,0
+815280007,0
+815280008,0
+815280009,0
+815280010,0
+815280011,0
+815280012,0
+815280013,0
+815280014,0
+815280015,0
+815280016,0
+815280017,0
+815280018,0
+866540001,0
+866540002,0
+866540003,0
+866540004,0
+866540005,0
+866540006,0
+866540007,0
+866540008,0
+866540009,0
+866540010,0
+866540011,0
+898080001,0
+898080002,0
+898080003,0
+898080004,0
+898080005,0
+898080006,0
+898080007,0
+898080008,0
+898080009,0
+898080010,0
+898080011,0
+898080012,0
+898080013,0
+898080014,0
+898080015,0
+898080016,0
+898080017,0
+898080018,0
+898080019,0
+898080020,0
+898080021,0
+898080022,0
+898080023,0
+898080024,0
+898080025,0
+898080026,0
+898080027,0
+898080028,0
+898080029,0
+898080030,0
+898080031,0
+898080032,0
+898080033,0
+898080034,0
+898080035,0
+898080036,0
+910190001,0
+910190002,0
+910190003,0
+910190004,0
+910190005,0
+910190006,0
+910190007,0
+954160001,0
+954160002,0
+982710001,0
+982710002,0
+982710003,0
+982710004,0
+982710005,0
+982710006,0
+982710007,0
+982710008,0
+982710009,0
+982710010,0
+982710011,0
+982710012,0
+982710013,0
+982710014,0
+982710015,0
+1026760001,0
+1026760002,0
+1026760003,0
+1026760004,0
+1026760005,0
+1026760006,0
+1026760007,0
+1026760008,0
+1026760009,0
+1026760010,0
+1026760011,0
+1026760012,0
+1026760013,0
+1026760014,0
+1026760015,0
+1026760016,0
+1026760017,0
+1026760018,0
+1026760019,0
+1026760020,0
+1026760021,0
+1026760022,0
+1026760023,0
+1026760024,0
+1026760025,0
+1026760026,0
+1026760027,0
+1033960001,0
+1033960002,0
+1033960003,0
+1033960004,0
+1033960005,0
+1033960006,0
+1063530001,0
+1063530002,0
+1063530003,0
+1063530004,0
+1063530005,0
+1063530006,0
+1063530007,0
+1063530008,0
+1063530009,0
+1063530010,0
+1063530011,0
+1063530012,0
+1063530013,0
+1063530014,0
+1063530015,0
+1063530016,0
+1063530017,0
+1063530018,0
+1063530019,0
+1063530020,0
+1063530021,0
+1063530022,0
+1063530023,0
+1063530024,0
+1063530025,0
+1063530026,0
+1063530027,0
+1063530028,0
+1063530029,0
+1063530030,0
+1063530031,0
+1063530032,0
+1063530033,0
+1063530034,0
+1063530035,0
+1063530036,0
+1063530037,0
+1063530038,0
+1063530039,0
+1063530040,0
+1063530041,0
+1063530042,0
+1063530043,0
+1063530044,0
+1063530045,0
+1063530046,0
+1063530047,0
+1063530048,0
+1063530049,0
+1063530050,0
+1063530051,0
+1063530052,0
+1063530053,0
+1063530054,0
+1063530055,0
+1063530056,0
+1063530057,0
+1063530058,0
+1063530059,0
+1063530060,0
+1063530061,0
+1063530062,0
+1063530063,0
+1063530064,0
+1063530065,0
+1063530066,0
+1063530067,0
+1063530068,0
+1063530069,0
+1063530070,0
+1063530071,0
+1063530072,0
+1063530073,0
+1063530074,0
+1063530075,0
+1063530076,0
+1063530077,0
+1063530078,0
+1063530079,0
+1063530080,0
+1063530081,0
+1063530082,0
+1063530083,0
+1113370001,0
+1113370002,0
+1113370003,0
+1113370004,0
+1157070001,0
+1157070002,0
+1157070003,0
+1157070004,0
+1157070005,0
+1157070006,0
+1157070007,0
+1157070008,0
+1172280001,0
+1172280002,0
+1172280003,0
+1172280004,0
+1172280005,0
+1172280006,0
+1172280007,0
+1172280008,0
+1172280009,0
+1210650001,0
+1210650002,0
+1210650003,0
+1210650004,0
+1210650005,0
+1210650006,0
+1210650007,0
+1210650008,0
+1210650009,0
+1210650010,0
+1210650011,0
+1210650012,0
+1210650013,0
+1210650014,0
+1210650015,0
+1210650016,0
+1210650017,0
+1210650018,0
+1210650019,0
+1210650020,0
+1210650021,0
+1210650022,0
+1210650023,0
+1210650024,0
+1210650025,0
+1210650026,0
+1210650027,0
+1210650028,0
+1217930001,0
+1217930002,0
+1217930003,0
+1217930004,0
+1217930005,0
+1245640001,0
+1245640002,0
+1245640003,0
+1245640004,0
+1245640005,0
+1245640006,0
+1245640007,0
+1245640008,0
+1245640009,0
+1245640010,0
+1245640011,0
+1245640012,0
+1245640013,0
+1245640014,0
+1245640015,0
+1245640016,0
+1245640017,0
+1245640018,0
+1245640019,0
+1245640020,0
+1245640021,0
+1245640022,0
+1245640023,0
+1245640024,0
+1245640025,0
+1245640026,0
+1245640027,0
+1245640028,0
+1245640029,0
+1245640030,0
+1245640031,0
+1245640032,0
+1245640033,0
+1245640034,0
+1245640035,0
+1245640036,0
+1245640037,0
+1245640038,0
+1248270001,0
+1248270002,0
+1248270003,0
+1248270004,0
+1248270005,0
+1248270006,0
+1248270007,0
+1248270008,0
+1248270009,0
+1248270010,0
+1248270011,0
+1248270012,0
+1248270013,0
+1250210001,0
+1250210002,0
+1250210003,0
+1250210004,0
+1250210005,0
+1250210006,0
+1250210007,0
+1250210008,0
+1250210009,0
+1250210010,0
+1250210011,0
+1250210012,0
+1269890001,0
+1269890002,0
+1269890003,0
+1269890004,0
+1269890005,0
+1269890006,0
+1269890007,0
+1269890008,0
+1269890009,0
+1269890010,0
+1269890011,0
+1269890012,0
+1269890013,0
+1269890014,0
+1269890015,0
+1295570001,0
+1298890001,0
+1298890002,0
+1298890003,0
+1298890004,0
+1298890005,0
+1298890006,0
+1298890007,0
+1298890008,0
+1298890009,0
+1298890010,0
+1298890012,0
+1298890013,0
+1298890014,0
+1298890015,0
+1298890016,0
+1298890017,0
+1298890018,0
+1298890019,0
+1298890020,0
+1298890021,0
+1334900001,0
+1334900002,0
+1334900003,0
+1334900004,0
+1334900005,0
+1334900006,0
+1337530001,0
+1337530002,0
+1337530003,0
+1337530004,0
+1337530005,0
+1337530006,0
+1337530007,0
+1337530008,0
+1337530009,0
+1337530010,0
+1337530011,0
+1337530012,0
+1337530013,0
+1337530014,0
+1337530015,0
+1337530016,0
+1337530017,0
+1337530018,0
+1337530019,0
+1337530020,0
+1337530021,0
+1337530022,0
+1337530023,0
+1337530024,0
+1337530025,0
+1337530026,0
+1337530027,0
+1337530028,0
+1337530029,0
+1337530030,0
+1337530031,0
+1337530032,0
+1337530033,0
+1337530034,0
+1337530035,0
+1337530036,0
+1337530037,0
+1337530038,0
+1337530039,0
+1337530040,0
+1337530041,0
+1337530042,0
+1337530043,0
+1337530044,0
+1337530045,0
+1337530046,0
+1337530047,0
+1337530048,0
+1337530049,0
+1337530050,0
+1337530051,0
+1337530052,0
+1337530053,0
+1337530054,0
+1337530055,0
+1337530056,0
+1337530057,0
+1337530058,0
+1337530059,0
+1337530060,0
+1337530061,0
+1337530062,0
+1337530063,0
+1337530064,0
+1337530065,0
+1337530066,0
+1337530067,0
+1337530068,0
+1337530069,0
+1337530070,0
+1337530071,0
+1337530072,0
+1337530073,0
+1337530074,0
+1337530075,0
+1337530076,0
+1337530077,0
+1337530078,0
+1337530079,0
+1337530080,0
+1337530081,0
+1337530082,0
+1382350001,0
+1382350002,0
+1382350003,0
+1382350004,0
+1382350005,0
+1382350006,0
+1382350007,0
+1382350008,0
+1394210001,0
+1394210002,0
+1394210003,0
+1394210004,0
+1394210005,0
+1394210006,0
+1394210007,0
+1394210008,0
+1394210009,0
+1394210010,0
+1394410001,0
+1394410002,0
+1394410003,0
+1394410004,0
+1394410005,0
+1404360001,0
+1404360002,0
+1404360003,0
+1404360004,0
+1404360005,0
+1404360006,0
+1404360007,0
+1404360009,0
+1404360010,0
+1404360011,0
+1404360013,0
+1404360015,0
+1404360016,0
+1404360018,0
+1404360019,0
+1404360022,0
+1404360023,0
+1404360024,0
+1404360025,0
+1404360026,0
+1404360027,0
+1404360029,0
+1453730001,0
+1453730002,0
+1453730003,0
+1453730004,0
+1453730005,0
+1453730006,0
+1453730007,0
+1453730008,0
+1453730009,0
+1453730010,0
+1453730011,0
+1453730012,0
+1453730013,0
+1453730014,0
+1456280001,0
+1456280002,0
+1456280003,0
+1456280004,0
+1480650001,0
+1480650002,0
+1480650003,0
+1480650004,0
+1480650005,0
+1480650006,0
+1480650007,0
+1480650008,0
+1481400001,0
+1481400002,0
+1481400003,0
+1485260001,0
+1485260002,0
+1485260003,0
+1485260004,0
+1485260005,0
+1486660001,0
+1486660002,0
+1486660003,0
+1486660004,0
+1486660005,0
+1486660006,0
+1486660007,0
+1486660008,0
+1486660009,0
+1486660010,0
+1486660011,0
+1486660012,0
+1486660013,0
+1486660014,0
+1550280001,0
+1550280002,0
+1550280003,0
+1550280004,0
+1550280005,0
+1550280006,0
+1550280007,0
+1550280008,0
+1550280009,0
+1550280010,0
+1550280011,0
+1550280012,0
+1550280013,0
+1550280014,0
+1550280015,0
+1550280016,0
+1550280017,0
+1550280018,0
+1550280019,0
+1550280020,0
+1550280021,0
+1550280022,0
+1550280023,0
+1550280024,0
+1550280025,0
+1550280026,0
+1550280027,0
+1550280028,0
+1550280029,0
+1550280030,0
+1550280031,0
+1550280032,0
+1550280033,0
+1550280034,0
+1550280035,0
+1550280036,0
+1550280037,0
+1550280038,0
+1550280039,0
+1550280040,0
+1550280041,0
+1550280042,0
+1550280043,0
+1550280044,0
+1550280045,0
+1550280046,0
+1550280047,0
+1550280048,0
+1550280049,0
+1550280050,0
+1550280051,0
+1550280052,0
+1550280053,0
+1550280054,0
+1550280055,0
+1561560001,0
+1561560002,0
+1561560003,0
+1561560004,0
+1561560005,0
+1561560006,0
+1561560007,0
+1561560008,0
+1561560009,0
+1561560010,0
+1561560011,0
+1561560012,0
+1561560013,0
+1561560014,0
+1561560015,0
+1561560016,0
+1561560017,0
+1561560018,0
+1561560019,0
+1561560020,0
+1561560021,0
+1561560022,0
+1561560023,0
+1561560024,0
+1561560025,0
+1561560026,0
+1561560027,0
+1561560028,0
+1561560029,0
+1561560030,0
+1561560031,0
+1561560032,0
+1595490001,0
+1595490003,0
+1595490004,0
+1595490005,0
+1595490006,0
+1595490007,0
+1595490008,0
+1595490009,0
+1595490010,0
+1595490011,0
+1595490012,0
+1595490013,0
+1595490014,0
+1595490015,0
+1595490016,0
+1595490017,0
+1595490018,0
+1595490019,0
+1595490020,0
+1601810001,0
+1601810002,0
+1601810003,0
+1612820001,0
+1612820002,0
+1652800001,0
+1652800002,0
+1652800003,0
+1652800004,0
+1652800005,0
+1652800006,0
+1652800007,0
+1652800008,0
+1652800009,0
+1652800010,0
+1652800011,0
+1652800012,0
+1652800013,0
+1676620001,0
+1676620002,0
+1676620003,0
+1676620004,0
+1676620005,0
+1676620006,0
+1676620007,0
+1695870001,0
+1695870002,0
+1695870003,0
+1695870004,0
+1695870005,0
+1695870006,0
+1705030001,0
+1705030002,0
+1705030003,0
+1705030004,0
+1705030005,0
+1705030006,0
+1705030007,0
+1705030008,0
+1705030009,0
+1705030010,0
+1705030011,0
+1705030012,0
+1705030013,0
+1705030014,0
+1705030015,0
+1705030016,0
+1707840001,0
+1707840002,0
+1707840003,0
+1707840004,0
+1707840005,0
+1707840006,0
+1707840007,0
+1707840008,0
+1801560001,0
+1801560002,0
+1801560003,0
+1801560004,0
+1801560005,0
+1801560006,0
+1801560007,0
+1801560008,0
+1801560009,0
+1801560010,0
+1801560011,0
+1906880001,0
+1906880002,0
+1906880003,0
+1931980001,0
+1931980002,0
+1931980003,0
+1931980004,0
+1931980005,0
+1931980006,0
+1931980007,0
+1931980008,0
+1931980009,0
+1931980010,0
+1931980011,0
+1931980012,0
+1931980013,0
+1931980014,0
+2020950001,0
+2020950002,0
+2020950003,0
+2077870001,0
+2077870002,0
+2077870003,0
+2077870004,0
+2077870005,0
+2163140001,0
+2163140002,0
+2163140003,0
+2163140004,0
+2163140005,0
+2163140006,0
+2163140007,0
+2163140008,0
+2163140009,0
+2163140010,0
+2163140011,0
+2163140012,0
+2163140013,0
+2163140014,0
+2163140015,0
+2163140016,0
+2163140017,0
+2163140018,0
+2163140019,0
+2163140020,0
+2163140021,0
+2163140022,0
+2163140023,0
+2163140024,0
+2163140025,0
+2163140026,0
+2163140027,0
+2163140028,0
+2163140029,0
+2163140030,0
+2163140031,0
+2163140032,0
+2163140033,0
+2163140034,0
+2163140035,0
+2163140036,0
+2163140037,0
+2163140038,0
+2163140039,0
+2163140040,0
+2163140041,0
+2163140042,0
+2163140043,0
+2163140044,0
+2163140045,0
+2163140046,0
+2163140047,0
+2163140048,0
+2163140049,0
+2163140050,0
+2163140051,0
+2163140052,0
+2163140053,0
+2163140054,0
+2163140055,0
+2163140056,0
+2163140057,0
+2163140058,0
+2193270001,0
+2293180001,0
+2293180002,0
+2293180003,0
+438100001,1
+438100002,1
+438100003,1
+438100004,1
+438100005,1
+438100006,1
+438100007,1
+438100008,1
+438100009,1
+438100010,1
+529150001,1
+529150002,1
+529150003,1
+529150004,1
+533970001,1
+533970002,1
+533970003,1
+600140001,1
+600140002,1
+600140003,1
+600140004,1
+600140005,1
+600140006,1
+600140007,1
+600140008,1
+600140009,1
+600140010,1
+600140011,1
+600140012,1
+600140013,1
+600140014,1
+600140015,1
+600140016,1
+600140017,1
+600140018,1
+600140019,1
+600140020,1
+600140021,1
+600140022,1
+600140023,1
+600140024,1
+631660001,1
+714100001,1
+714100002,1
+714100003,1
+714100004,1
+714100005,1
+714100006,1
+714100007,1
+714100008,1
+1178780001,1
+1178780002,1
+1178780003,1
+1178780004,1
+1178780005,1
+1178780006,1
+1178780007,1
+1178780008,1
+1178780009,1
+1178780010,1
+1178780011,1
+1178780012,1
+1178780013,1
+1178780014,1
+1178780015,1
+1178780016,1
+1337060001,1
+1337060002,1
+1337060003,1
+1337060004,1
+1337060005,1
+1337060006,1
+1337060007,1
+1337060008,1
+1337060009,1
+1337060010,1
+1337060011,1
+1337060012,1
+1337060013,1
+1337060014,1
+1337060015,1
+1337060016,1
+1337060017,1
+1337060018,1
+1337060019,1
+1337060020,1
+1337060021,1
+1337060022,1
+1337060023,1
+1337060024,1
+1337060025,1
+1337060026,1
+1337060027,1
+1337060028,1
+1337060029,1
+1337060030,1
+1337060031,1
+1337060032,1
+1337060033,1
+1337060034,1
+1337060035,1
+1337060036,1
+1337060039,1
+1337060040,1
+1337060041,1
+1960620001,1
+1960620002,1
+1960620008,1
+1960620010,1
+1960620011,1
+1960620012,1
+1960620013,1
+1960620014,1
+1960620015,1
+1960620016,1
+2089520001,1
+2089520002,1
+2089520003,1
+2089520004,1
+2308940001,1
+2308940002,1
+2308940003,1
+2308940004,1
+2308940005,1
+2308940006,1
+2308940007,1
+2308940008,1
+2308940009,1
+2308940010,1
+2308940011,1
+2308940012,1
+2308940013,1
+2308940014,1
+2308940015,1
+2308940016,1
+2308940017,1
+269170001,2
+269170002,2
+269170003,2
+269170004,2
+269170005,2
+269170006,2
+269170007,2
+269170008,2
+269170009,2
+451980001,2
+451980002,2
+451980003,2
+451980004,2
+451980005,2
+451980006,2
+451980007,2
+451980008,2
+451980009,2
+451980010,2
+451980011,2
+451980012,2
+451980013,2
+451980014,2
+451980015,2
+451980016,2
+451980017,2
+451980018,2
+457550001,2
+457550002,2
+457550003,2
+457550004,2
+457550005,2
+490250001,2
+490250002,2
+490250003,2
+490250004,2
+490250005,2
+490250006,2
+490250007,2
+490250008,2
+490250009,2
+490250010,2
+490250011,2
+490250012,2
+490250013,2
+490250014,2
+490250015,2
+490250016,2
+490250017,2
+490250018,2
+490250019,2
+497820001,2
+497820002,2
+497820003,2
+591680001,2
+591680002,2
+591680003,2
+591680004,2
+591680005,2
+591680006,2
+591680007,2
+591680008,2
+591680009,2
+591680010,2
+591680011,2
+591680012,2
+591680013,2
+731790001,2
+731790002,2
+731790003,2
+731790004,2
+731790005,2
+731790006,2
+731790007,2
+731790008,2
+731790009,2
+731790010,2
+758210001,2
+758210002,2
+758210003,2
+758210004,2
+758210005,2
+758210006,2
+758210007,2
+758210008,2
+758210009,2
+758210010,2
+758210011,2
+758210012,2
+758210013,2
+758210014,2
+891960001,2
+891960002,2
+891960003,2
+891960004,2
+891960005,2
+891960006,2
+891960007,2
+891960008,2
+891960009,2
+891960010,2
+997760001,2
+997760002,2
+997760003,2
+997760004,2
+997760005,2
+997760006,2
+997760007,2
+997760008,2
+997760009,2
+997760010,2
+997760011,2
+997760012,2
+998660001,2
+998660002,2
+998660003,2
+998660004,2
+998660005,2
+1150310001,2
+1150310002,2
+1150310003,2
+1150310004,2
+1150310005,2
+1150310006,2
+1264160001,2
+1264160002,2
+1264160003,2
+1264160004,2
+1264160005,2
+1264160006,2
+1264160007,2
+1264160008,2
+1264160009,2
+1264160010,2
+1264160011,2
+1264160012,2
+1264160013,2
+1264160014,2
+1264160015,2
+1264160016,2
+1264160017,2
+1270910001,2
+1270910002,2
+1270910003,2
+1270910004,2
+1270910005,2
+1388030001,2
+1488730001,2
+1512840001,2
+1512840002,2
+1512840003,2
+1512840004,2
+1512840005,2
+1512840006,2
+1512840007,2
+1730290001,2
+1730290002,2
+1730290003,2
+1730290004,2
+1730290005,2
+1805510001,2
+1805510002,2
+1805510003,2
+1805510004,2
+1805510005,2
+1805510006,2
+1805510007,2
+1805510008,2
+1805510009,2
+1805510010,2
+1805510011,2
+1805510012,2
+1805510013,2
+1805510014,2
+1805510015,2
+1805510016,2
+1805510017,2
+1805510018,2
+1805510019,2
+1805510020,2
+1805510021,2
+1805510022,2
+1805510023,2
+1805510024,2
+1805510025,2
+1805510026,2
+1805510027,2
+1805510028,2
+1805510029,2
+1805510030,2
+1805510031,2
+1805510032,2
+1805510033,2
+1825460001,2
+1825460002,2
+1825460003,2
+1825460004,2
+1825460005,2
+1825460006,2
+1825460007,2
+1825460008,2
+1825460009,2
+1825460010,2
+1825460011,2
+1825460012,2
+1862180001,2
+1862180002,2
+1862180003,2
+1862180004,2
+1862180005,2
+1862180006,2
+1862180007,2
+2025000001,2
+2025000002,2
+2025000003,2
+2025000004,2
+2025000005,2
+2025000006,2
+2025000007,2
+2025000008,2
+2025000009,2
+2025000010,2
+2057000001,2
+2057000002,2
+2057000003,2
+2057000004,2
+2057000005,2
+2057000006,2
+2057000007,2
+2057000008,2
+2057000009,2
+2057000010,2
+2057000011,2
+2057000012,2
+2057000013,2
+2057000014,2
+2057000015,2
+2057000016,2
+2057000017,2
+2057000018,2
+2057000019,2
+2057000020,2
+2057000021,2
+2057000022,2
+2057000023,2
+2057000024,2
+2057000025,2
+2057000026,2
+2057000027,2
+2057000028,2
+2057000029,2
+2057000030,2
+2057000031,2
+2057000032,2
+2057000033,2
+2057000034,2
+2057000035,2
+2057000036,2
+2057000037,2
+2057000038,2
+2057000039,2
+2057000040,2
+2057000041,2
+2057000042,2
+2057000043,2
+2057000044,2
+2057000045,2
+2057000046,2
+2057000047,2
+2057000048,2
+2057000049,2
+2057000050,2
+2057000051,2
+2057000052,2
+2057000053,2
+2057000054,2
+2057000055,2
+2057000056,2
+2057000057,2
+2057000058,2
+2057000059,2
+2057000060,2
+2057000061,2
+2057000062,2
+2057000063,2
+2057000064,2
+2057000065,2
+2057000066,2
+2057000067,2
+2057000068,2
+2057000069,2
+2057000070,2
+2057000071,2
+2057000072,2
+2057000073,2
+2057000074,2
+2057000075,2
+2057000076,2
+2057000077,2
+2057000078,2
+2057000079,2
+2057000080,2
+2057000081,2
+2057000082,2
+2057000083,2
+2057000084,2
+2057000085,2
+2057000086,2
+2057000087,2
+2057000088,2
+2057000089,2
+2057000090,2
+2057000091,2
+2057000092,2
+2057000093,2
+2057000094,2
+2057000095,2
+2057000096,2
+2057000097,2
+2057000098,2
+2057000099,2
+2057000100,2
+2057000101,2
+2057000102,2
+2057000103,2
+2057000104,2
+2057000105,2
+2057000106,2
+2057000107,2
+2057000108,2
+2057000109,2
+2057000110,2
+2057000111,2
+2057000112,2
+2057000113,2
+2057000114,2
+2057000115,2
+2057000116,2
+2057000117,2
+2057000118,2
+2057000119,2
+2057000120,2
+2057000121,2
+2057000122,2
+2057000123,2
+2057000124,2
+2057000125,2
+2057000126,2
+2057000127,2
+2057000128,2
+2057000129,2
+2057000130,2
+2057000131,2
+2057000132,2
+2057000133,2
+2057000134,2
+2057000135,2
+2057000136,2
+2057000137,2
+2057000138,2
+2057000139,2
+2057000140,2
+2057000141,2
+2057000142,2
+2057000143,2
+2057000144,2
+2057000145,2
+2057000146,2
+2057000147,2
+2057000148,2
+2057000149,2
+2057000150,2
+2057000151,2
+2057000152,2
+2057000153,2
+2057000154,2
+2057000155,2
+2057000156,2
+2057000157,2
+2057000158,2
+2057000159,2
+2057000160,2
+2057000161,2
+2057000162,2
+2057000163,2
+2057000164,2
+2057000165,2
+2057000166,2
+2057000167,2
+2057000168,2
+2057000169,2
+2057000170,2
+2057000171,2
+2057000172,2
+2057000173,2
+2057000174,2
+2057000175,2
+2057000176,2
+2057000177,2
+2057000178,2
+2224020001,2
+2224020002,2
+2224020003,2
+2224020004,2
+2224020005,2
+2224020006,2
+2224020007,2
+2224020008,2
+2224020009,2
+2224020010,2
+2224020011,2
+2224020012,2
+2224020013,2
+2224020014,2
+2224020015,2
+2224020016,2
+2224020017,2
+2224020018,2
+2224020019,2
+2224020020,2
+2224020021,2
+2224020022,2
+2224020023,2
+2224020024,2
+2224020025,2
+2224020026,2
+2224020027,2
+2224020028,2
+2224020029,2
+2224020030,2
+2224020031,2
+2224020032,2
+2224020033,2
+2224020034,2
+2224020035,2
+2224020036,2
+2224020037,2
+2224020038,2
+2224020039,2
+2224020040,2
+2224020041,2
+2224020042,2
+2224020043,2
+2224020044,2
+2224020045,2
+2224020046,2
+2224020047,2
+2224020048,2
+2224020049,2
+2224020050,2
+2224020051,2
+2224020052,2
+2224020053,2
+2224020054,2
+2224020055,2
+2224020056,2
+2224020057,2
+2224020058,2

dataset/gen_cat_fold.py

@@ -0,0 +1,58 @@
+import json
+import random
+import pandas as pd
+
+DATASET_PATH = './data/coco_det/annotations/semantics.json'
+CONTEXT_DATASET_PATH = './data/coco_context/annotations/context.json'
+
+
+def parse_img_id(img_id):
+    app_id = int(str(img_id)[:-3])
+    img = int(str(img_id)[-3:])
+    return app_id, img
+
+with open(DATASET_PATH, 'r') as f:
+    dataset = json.load(f)
+
+cat_name_cat = {cat['name']: cat for cat in dataset['categories']}
+
+img_ids = set([img['id'] for img in dataset['images']])
+
+app_id_img = {}
+
+for img in dataset['images']:
+    app_id, img = parse_img_id(img['id'])
+    if app_id not in app_id_img:
+        app_id_img[app_id] = []
+    app_id_img[app_id].append(img)
+
+with open(CONTEXT_DATASET_PATH, 'r') as f:
+    context_dataset = json.load(f)
+
+context_cat_name = [cat['name'] for cat in context_dataset['categories'] if not cat['name'].endswith('-n')]
+context_cat_id = [cat['id'] for cat in dataset['categories'] if cat['name'] in context_cat_name]
+
+context_related_img_ids = []
+
+for ann in dataset['annotations']:
+    if ann['category_id'] in context_cat_id:
+        context_related_img_ids.append(ann['image_id'])
+
+context_related_img_ids = set(context_related_img_ids)
+
+# Get the images that are not already in the context_related_img set
+remaining_img_ids = img_ids - context_related_img_ids
+
+# Split the remaining images into two groups with a ratio of 6:1
+num_train = int(len(remaining_img_ids) * 6 / 7)
+train_images = random.sample(remaining_img_ids, num_train)
+val_images = remaining_img_ids - set(train_images)
+
+print(len(train_images), len(val_images), len(context_related_img_ids))
+
+df_data = []
+df_data.extend([[img_id, 0] for img_id in train_images])
+df_data.extend([[img_id, 1] for img_id in val_images])
+df_data.extend([[img_id, 2] for img_id in context_related_img_ids])
+df = pd.DataFrame(df_data, columns=['img_id', 'fold'])
+df.to_csv('./fold_context.csv', index=False)

dataset/get_union_test.py

@@ -0,0 +1,52 @@
+import json
+
+FORMAT = 'det'
+
+TASK = 'interactable'
+# TASK = 'semantics'
+# TASK = 'interaction'
+
+DATASET_PATH = f'./data/coco_{FORMAT}/annotations/{TASK}.json'
+TEST_613_PATH = f'../evaluation/gts/{FORMAT}/{TASK}/613.json'
+TEST_GENRE_PATH = f'../evaluation/gts/{FORMAT}/{TASK}/genre.json'
+TEST_CAT_PATH = f'../evaluation/gts/{FORMAT}/{TASK}/cat.json'
+UNION_PATH = f'../evaluation/gts/{FORMAT}/{TASK}/union_test.json'
+UNION3_PATH = f'../evaluation/gts/{FORMAT}/{TASK}/union3_test.json'
+
+
+def get_union_test(fullset, testset1, testset2):
+    
+    union_test = {'images': [], 'categories': fullset['categories'], 'annotations': []}
+    
+    img_ids = []
+    
+    for img in testset1['images']:
+        img_ids.append(img['id'])
+        union_test['images'].append(img)
+    for img in testset2['images']:
+        if img['id'] not in img_ids:
+            union_test['images'].append(img)
+            img_ids.append(img['id'])
+    
+    for ann in fullset['annotations']:
+        if ann['image_id'] in img_ids:
+            union_test['annotations'].append(ann)
+    
+    return union_test
+
+
+with open(DATASET_PATH, 'r') as f:
+    dataset = json.load(f)
+with open(TEST_613_PATH, 'r') as f:
+    test_613 = json.load(f)
+with open(TEST_GENRE_PATH, 'r') as f:
+    test_genre = json.load(f)
+with open(TEST_CAT_PATH, 'r') as f:
+    test_cat = json.load(f)
+
+union_test = get_union_test(dataset, test_613, test_genre)
+with open(UNION_PATH, 'w') as f:
+    json.dump(union_test, f, indent=4)
+union3_test = get_union_test(dataset, union_test, test_cat)
+with open(UNION3_PATH, 'w') as f:
+    json.dump(union3_test, f, indent=4)

dataset/get_val_gt.py

@@ -0,0 +1,66 @@
+import os
+import shutil
+
+# SPLIT = '613'
+# TRAIN_FOLD = '0,1,4,5,7,9'
+# VAL_FOLD = '3'
+# TEST_FOLD = '2,6,8'
+# SPLIT_FILE = 'fold_app.csv'
+
+# SPLIT = 'genre'
+# TRAIN_FOLD = '0'
+# VAL_FOLD = '1'
+# TEST_FOLD = '2'
+# SPLIT_FILE = 'fold_genre.csv'
+
+SPLIT = 'cat'
+TRAIN_FOLD = '0'
+VAL_FOLD = '1'
+TEST_FOLD = '2'
+SPLIT_FILE = 'fold_cat.csv'
+
+INPUT_PATH = './data'
+OUTPUT_PATH = '../evaluation/gts'
+TASKS = ['interaction', 'semantics', 'interactable']
+FORMATS = ['det']
+split_script_file = './split_coco.py'
+# fold_file = './fold.csv' # split on image
+
+
+for task in TASKS:
+    for format in FORMATS:
+        ann_file = os.path.join(INPUT_PATH, f'coco_{format}/annotations/{task}.json')
+        img_dir = os.path.join(INPUT_PATH, 'coco_merged/images', task)
+        split_path = os.path.join(INPUT_PATH, 'coco_split')
+        if os.path.exists(split_path):
+            shutil.rmtree(split_path)
+        # K fold
+        # for test_fold in range(FOLD_CNT):
+        #     train_fold = ','.join([str(i) for i in range(FOLD_CNT) if i != test_fold])
+        #     cli = f'python {split_script_file} ' + \
+        #             f'--ann_file {ann_file} ' + \
+        #             f'--img_dir {img_dir} ' + \
+        #             f'--output_path {split_path} ' + \
+        #             f'--fold_file {fold_file} ' + \
+        #             f'--train_fold {train_fold} ' + \
+        #             f'--val_fold {test_fold}'
+        #     print(cli)
+        #     os.system(cli)
+        #     os.makedirs(os.path.join(OUTPUT_PATH, f'{format}/{task}'), exist_ok=True)
+        #     shutil.copyfile(os.path.join(split_path, 'annotations', 'instances_val2017.json'), os.path.join(OUTPUT_PATH, f'{format}/{task}/fold{test_fold}.json'))
+        #     shutil.rmtree(split_path)
+
+        # custom split
+        cli = f'python {split_script_file} ' + \
+            f'--ann_file {ann_file} ' + \
+            f'--img_dir {img_dir} ' + \
+            f'--output_path {split_path} ' + \
+            f'--fold_file {SPLIT_FILE} ' + \
+            f'--train_fold {TRAIN_FOLD} ' + \
+            f'--val_fold {VAL_FOLD} ' + \
+            f'--test_fold {TEST_FOLD} '
+        print(cli)
+        os.system(cli)
+        os.makedirs(os.path.join(OUTPUT_PATH, f'{format}/{task}'), exist_ok=True)
+        shutil.copy2(os.path.join(split_path, 'annotations', 'instances_test2017.json'), os.path.join(OUTPUT_PATH, f'{format}/{task}/{SPLIT}.json'))
+        shutil.rmtree(split_path)

dataset/interactable_coco.py

@@ -0,0 +1,19 @@
+# obtain interactable dataset from semantics dataset
+import json
+import os
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--input', type=str, required=True)
+parser.add_argument('--output', type=str, required=True)
+args = parser.parse_args()
+
+with open(args.input, 'r') as fr:
+    data = json.load(fr)
+
+data['categories'] = [{'id': 1, 'name': 'interactable'}]
+for annotation in data['annotations']:
+    annotation['category_id'] = 1
+
+with open(args.output, 'w') as fw:
+    json.dump(data, fw, indent=4)

dataset/list-union3.txt

@@ -0,0 +1 @@
+438100001, 438100002, 438100003, 438100004, 438100005, 438100006, 438100007, 438100008, 438100009, 438100010, 457380008, 457380010, 457380013, 1250210001, 1250210002, 1250210003, 1250210004, 1250210005, 1250210006, 1250210007, 1250210008, 1250210009, 1250210010, 1250210011, 1250210012, 1676620003, 1676620004, 1676620005, 1676620006, 1676620007, 1730290001, 1730290002, 1730290003, 1730290004, 1730290005, 497820001, 497820002, 497820003, 1178780015, 1486660012, 1456280001, 1456280002, 1456280003, 1456280004, 591680001, 591680002, 591680003, 591680004, 591680005, 591680006, 1707840007, 591680008, 591680009, 591680010, 591680011, 591680012, 591680013, 591680007, 1394410001, 1394410002, 1394410003, 1394410004, 1394410005, 622310001, 622310002, 716260002, 716260003, 716260005, 790750001, 790750002, 790750003, 790750004, 790750005, 790750006, 954160001, 954160002, 1561560001, 1561560002, 1561560003, 1561560004, 1561560005, 1561560006, 1561560007, 1561560008, 1561560009, 1561560010, 1561560011, 1561560012, 1561560013, 1561560014, 1561560015, 1561560016, 1561560017, 1561560018, 1561560019, 1561560020, 1561560021, 1561560022, 1561560023, 1561560024, 1561560025, 1561560026, 1561560027, 1561560028, 1561560029, 1561560030, 1561560031, 1561560032, 726910002, 726910003, 726910004, 726910005, 726910007, 726910010, 2057000001, 2057000002, 2057000003, 2057000004, 2057000005, 2057000006, 2057000007, 2057000008, 2057000009, 2057000010, 2057000011, 2057000012, 2057000013, 2057000014, 2057000015, 2057000016, 2057000017, 2057000018, 2057000019, 2057000020, 2057000021, 2057000022, 2057000023, 2057000024, 2057000025, 2057000026, 2057000027, 2057000028, 2057000029, 2057000030, 2057000031, 2057000032, 2057000033, 2057000034, 2057000035, 2057000036, 2057000037, 2057000038, 2057000039, 2057000040, 2057000041, 2057000042, 2057000043, 2057000044, 2057000045, 2057000046, 2057000047, 2057000048, 2057000049, 2057000050, 2057000051, 2057000052, 2057000053, 2057000054, 2057000055, 2057000056, 2057000057, 2057000058, 2057000059, 2057000060, 2057000061, 2057000062, 2057000063, 2057000064, 2057000065, 2057000066, 2057000067, 2057000068, 2057000069, 2057000070, 2057000071, 2057000072, 2057000073, 2057000074, 2057000075, 2057000076, 2057000077, 2057000078, 2057000079, 2057000080, 2057000081, 2057000082, 2057000083, 2057000084, 2057000085, 2057000086, 2057000087, 2057000088, 2057000089, 2057000090, 2057000091, 2057000092, 2057000093, 2057000094, 2057000095, 2057000096, 2057000097, 2057000098, 2057000099, 2057000100, 2057000101, 2057000102, 2057000103, 2057000104, 2057000105, 2057000106, 2057000107, 2057000108, 2057000109, 2057000110, 2057000111, 2057000112, 2057000113, 2057000114, 2057000115, 2057000116, 2057000117, 2057000118, 2057000119, 2057000120, 2057000121, 2057000122, 2057000123, 2057000124, 2057000125, 2057000126, 2057000127, 2057000128, 2057000129, 2057000130, 2057000131, 2057000132, 2057000133, 2057000134, 2057000135, 2057000136, 2057000137, 2057000138, 2057000139, 2057000140, 2057000141, 2057000142, 2057000143, 2057000144, 2057000145, 2057000146, 2057000147, 2057000148, 2057000149, 2057000150, 2057000151, 2057000152, 2057000153, 2057000154, 2057000155, 2057000156, 2057000157, 2057000158, 2057000159, 2057000160, 2057000161, 2057000162, 2057000163, 2057000164, 2057000165, 2057000166, 2057000167, 2057000168, 2057000169, 2057000170, 2057000171, 2057000172, 2057000173, 2057000174, 2057000175, 2057000176, 2057000177, 2057000178, 2193270001, 1334900003, 1334900004, 1334900005, 1334900006, 1550280001, 1404360004, 1245640005, 1245640006, 1404360005, 1404360006, 1245640009, 1404360007, 1404360009, 1404360010, 1550280010, 1245640014, 1245640015, 1245640016, 1404360016, 1404360018, 1550280018, 269170001, 1550280012, 1404360022, 1404360023, 269170002, 269170009, 269170008, 1550280027, 269170003, 1404360029, 1550280029, 1550280030, 1245640032, 1550280031, 1245640034, 1550280035, 1245640036, 1245640037, 1550280036, 1550280037, 1550280039, 1550280041, 1550280042, 1550280043, 1550280045, 1550280049, 1805510001, 1550280051, 1550280052, 1550280053, 1805510002, 1805510003, 1805510006, 1805510008, 1805510009, 1805510010, 1805510011, 1805510012, 1805510005, 1805510015, 1805510016, 1805510013, 1805510014, 1805510017, 1805510018, 1805510021, 1805510019, 1805510020, 1805510022, 1805510023, 1805510026, 1805510024, 1805510025, 1805510027, 1805510028, 1805510029, 1805510030, 1805510031, 1805510032, 1805510033, 2163140004, 2163140006, 2163140007, 2163140011, 1550280013, 2163140012, 2163140014, 2163140015, 2163140016, 2163140022, 2163140024, 2163140025, 2163140029, 2163140033, 2163140034, 2163140038, 2163140039, 2163140042, 2163140043, 2163140048, 2163140050, 2163140053, 2163140055, 2163140057, 2163140058, 1150310001, 1150310002, 1150310003, 1150310004, 1150310005, 1150310006, 1337530003, 1337530005, 1337530006, 1337530009, 1337530010, 1337530013, 1337530014, 1337530017, 1337530018, 1337530022, 1337530023, 1337530024, 1337530027, 1337530034, 1248270004, 1248270005, 1337530037, 1248270008, 1337530041, 1248270010, 1248270011, 1248270012, 1337530046, 1337530048, 1337530049, 1337530050, 1337530057, 1337530061, 1337530063, 1337530067, 1337530069, 1337530070, 1337530075, 1337530077, 1337530079, 982710005, 982710006, 982710008, 518580001, 518580002, 518580003, 518580004, 518580005, 518580006, 518580010, 518580011, 518580015, 518580016, 518580023, 518580024, 731790005, 731790010, 1488730001, 998660001, 998660002, 998660003, 998660004, 998660005, 2077870001, 2077870002, 2077870003, 2077870004, 2077870005, 720300003, 812460003, 812460004, 812460005, 812460006, 720300008, 812460007, 812460008, 812460009, 720300012, 812460010, 720300017, 1388030001, 269170004, 269170005, 269170006, 269170007, 457550001, 457550002, 457550003, 457550004, 457550005, 1480650002, 1453730001, 1453730002, 1453730003, 1453730004, 1453730005, 1453730006, 1453730007, 1453730008, 1453730009, 1453730010, 1453730011, 1453730012, 1453730013, 1453730014, 1480650007, 600140001, 600140002, 600140003, 600140004, 600140005, 600140006, 600140007, 600140008, 600140009, 600140010, 600140011, 600140012, 600140013, 600140014, 600140015, 600140016, 600140017, 600140018, 600140019, 600140020, 600140021, 600140022, 600140023, 600140024, 866540001, 866540002, 866540003, 866540004, 866540005, 866540006, 866540007, 866540008, 866540009, 866540010, 866540011, 1652800001, 1652800002, 1652800003, 1652800004, 1652800005, 1652800006, 1652800007, 1652800008, 1652800009, 1652800010, 1652800011, 1652800012, 1652800013, 2224020001, 2224020002, 2224020003, 2224020004, 2224020005, 2224020006, 2224020007, 2224020008, 2224020009, 2224020010, 2224020011, 2224020012, 2224020013, 2224020014, 2224020015, 2224020016, 2224020017, 2224020018, 2224020019, 2224020020, 2224020021, 2224020022, 2224020023, 2224020024, 2224020025, 2224020026, 2224020027, 2224020028, 2224020029, 2224020030, 2224020031, 2224020032, 2224020033, 2224020034, 2224020035, 2224020036, 2224020037, 2224020038, 2224020039, 2224020040, 2224020041, 2224020042, 2224020043, 2224020044, 2224020045, 2224020046, 2224020047, 2224020048, 2224020049, 2224020050, 2224020051, 2224020052, 2224020053, 2224020054, 2224020055, 2224020056, 2224020057, 2224020058, 1805510004, 1805510007, 1337060007, 1337060008, 1337060009, 1337060010, 1337060011, 1337060012, 1337060015, 731790001, 1337060018, 1337060019, 1337060020, 1337060021, 1337060022, 1337060023, 1337060024, 1337060025, 1337060026, 731790002, 731790003, 731790004, 1337060030, 731790006, 731790007, 731790008, 731790009, 1337060035, 891960001, 891960002, 891960003, 891960004, 891960005, 891960006, 891960007, 891960008, 891960009, 891960010, 451980001, 451980002, 451980003, 451980004, 451980005, 451980006, 451980007, 451980008, 451980009, 451980010, 451980011, 451980012, 451980013, 451980014, 451980015, 451980016, 451980017, 451980018, 1512840001, 1512840002, 1512840003, 1512840004, 1512840005, 1512840006, 1512840007, 2089520001, 2089520002, 2089520003, 2089520004, 1269890001, 1269890002, 1269890003, 1269890004, 1269890005, 1269890006, 1269890007, 1269890008, 1269890009, 1269890010, 1269890011, 1269890012, 1269890013, 1269890014, 1269890015, 997760001, 997760002, 997760003, 997760004, 997760005, 997760006, 997760007, 997760008, 997760009, 997760010, 997760011, 997760012, 1270910001, 1270910002, 1270910003, 1270910004, 1270910005, 2025000001, 2025000002, 2025000003, 2025000004, 2025000005, 2025000006, 660520005, 2025000007, 2025000009, 2025000010, 660520009, 2025000008, 1862180001, 1862180002, 1862180003, 1862180004, 1862180005, 1862180006, 1862180007, 1157070001, 1382350002, 1157070002, 1382350004, 1157070003, 1382350006, 1382350007, 1382350008, 1157070004, 1157070005, 1157070006, 1157070007, 1157070008, 1931980011, 1264160001, 1264160002, 1264160003, 1264160004, 1264160005, 1264160006, 1264160007, 1264160008, 1264160009, 1264160010, 1264160011, 1264160012, 1264160013, 1264160014, 1264160015, 1264160016, 1480650001, 898080018, 1480650003, 1298890004, 1480650004, 1298890006, 1480650005, 1298890008, 1298890009, 1298890010, 1480650006, 1298890012, 1298890013, 1298890014, 1480650008, 1298890016, 898080026, 898080027, 1298890019, 1298890020, 1298890021, 714100004, 714100006, 714100007, 1026760001, 1026760002, 1026760003, 1026760004, 1026760005, 1026760006, 1026760007, 1026760008, 1026760009, 1026760010, 1026760011, 1026760012, 1026760013, 1026760014, 1026760015, 1026760016, 1026760017, 1026760018, 1026760019, 1026760020, 1026760021, 1026760022, 1026760023, 1026760024, 1026760025, 1026760026, 1026760027, 1113370001, 1210650002, 1113370002, 1113370003, 1210650005, 1113370004, 1210650011, 1210650012, 1210650013, 1210650014, 528580003, 528580004, 1210650023, 1210650024, 1210650025, 1210650026, 1210650027, 1210650028, 1264160017, 758210001, 758210002, 758210003, 758210004, 758210005, 758210006, 758210007, 758210008, 758210009, 758210010, 758210011, 758210012, 758210013, 758210014, 1906880001, 1906880002, 1906880003, 1063530001, 1063530002, 1063530004, 1063530007, 1063530008, 1063530009, 1063530010, 1063530011, 1063530012, 1063530014, 1063530015, 1063530016, 1063530018, 1063530019, 1063530021, 1063530023, 1063530029, 1063530032, 1063530048, 1063530052, 898080019, 1063530057, 1063530059, 1063530065, 1063530067, 1063530069, 1063530070, 898080022, 1063530073, 1063530074, 1063530079, 1063530083, 898080028, 463290002, 463290003, 463290008, 1595490008, 490250001, 490250002, 490250003, 490250004, 490250005, 490250006, 490250007, 490250008, 490250009, 490250010, 490250011, 490250012, 490250013, 490250014, 490250015, 490250016, 490250017, 1825460002, 1825460003, 490250018, 490250019, 1825460001, 1825460004, 1825460005, 1825460009, 1825460006, 1825460007, 1825460008, 1825460010, 1825460011, 1825460012, 815280001, 815280002, 815280003, 815280004, 815280005, 815280006, 815280007, 815280008, 815280009, 815280010, 815280011, 815280012, 815280013, 815280014, 815280015, 815280016, 815280017, 815280018, 2020950001, 2020950002, 2020950003

dataset/list_union3.txt

@@ -0,0 +1 @@
+438100001, 438100002, 438100003, 438100004, 438100005, 438100006, 438100007, 438100008, 438100009, 438100010, 457380008, 457380010, 457380013, 1250210001, 1250210002, 1250210003, 1250210004, 1250210005, 1250210006, 1250210007, 1250210008, 1250210009, 1250210010, 1250210011, 1250210012, 1676620003, 1676620004, 1676620005, 1676620006, 1676620007, 1730290001, 1730290002, 1730290003, 1730290004, 1730290005, 497820001, 497820002, 497820003, 1178780015, 1486660012, 1456280001, 1456280002, 1456280003, 1456280004, 591680001, 591680002, 591680003, 591680004, 591680005, 591680006, 1707840007, 591680008, 591680009, 591680010, 591680011, 591680012, 591680013, 591680007, 1394410001, 1394410002, 1394410003, 1394410004, 1394410005, 622310001, 622310002, 716260002, 716260003, 716260005, 790750001, 790750002, 790750003, 790750004, 790750005, 790750006, 954160001, 954160002, 1561560001, 1561560002, 1561560003, 1561560004, 1561560005, 1561560006, 1561560007, 1561560008, 1561560009, 1561560010, 1561560011, 1561560012, 1561560013, 1561560014, 1561560015, 1561560016, 1561560017, 1561560018, 1561560019, 1561560020, 1561560021, 1561560022, 1561560023, 1561560024, 1561560025, 1561560026, 1561560027, 1561560028, 1561560029, 1561560030, 1561560031, 1561560032, 726910002, 726910003, 726910004, 726910005, 726910007, 726910010, 2057000001, 2057000002, 2057000003, 2057000004, 2057000005, 2057000006, 2057000007, 2057000008, 2057000009, 2057000010, 2057000011, 2057000012, 2057000013, 2057000014, 2057000015, 2057000016, 2057000017, 2057000018, 2057000019, 2057000020, 2057000021, 2057000022, 2057000023, 2057000024, 2057000025, 2057000026, 2057000027, 2057000028, 2057000029, 2057000030, 2057000031, 2057000032, 2057000033, 2057000034, 2057000035, 2057000036, 2057000037, 2057000038, 2057000039, 2057000040, 2057000041, 2057000042, 2057000043, 2057000044, 2057000045, 2057000046, 2057000047, 2057000048, 2057000049, 2057000050, 2057000051, 2057000052, 2057000053, 2057000054, 2057000055, 2057000056, 2057000057, 2057000058, 2057000059, 2057000060, 2057000061, 2057000062, 2057000063, 2057000064, 2057000065, 2057000066, 2057000067, 2057000068, 2057000069, 2057000070, 2057000071, 2057000072, 2057000073, 2057000074, 2057000075, 2057000076, 2057000077, 2057000078, 2057000079, 2057000080, 2057000081, 2057000082, 2057000083, 2057000084, 2057000085, 2057000086, 2057000087, 2057000088, 2057000089, 2057000090, 2057000091, 2057000092, 2057000093, 2057000094, 2057000095, 2057000096, 2057000097, 2057000098, 2057000099, 2057000100, 2057000101, 2057000102, 2057000103, 2057000104, 2057000105, 2057000106, 2057000107, 2057000108, 2057000109, 2057000110, 2057000111, 2057000112, 2057000113, 2057000114, 2057000115, 2057000116, 2057000117, 2057000118, 2057000119, 2057000120, 2057000121, 2057000122, 2057000123, 2057000124, 2057000125, 2057000126, 2057000127, 2057000128, 2057000129, 2057000130, 2057000131, 2057000132, 2057000133, 2057000134, 2057000135, 2057000136, 2057000137, 2057000138, 2057000139, 2057000140, 2057000141, 2057000142, 2057000143, 2057000144, 2057000145, 2057000146, 2057000147, 2057000148, 2057000149, 2057000150, 2057000151, 2057000152, 2057000153, 2057000154, 2057000155, 2057000156, 2057000157, 2057000158, 2057000159, 2057000160, 2057000161, 2057000162, 2057000163, 2057000164, 2057000165, 2057000166, 2057000167, 2057000168, 2057000169, 2057000170, 2057000171, 2057000172, 2057000173, 2057000174, 2057000175, 2057000176, 2057000177, 2057000178, 2193270001, 1334900003, 1334900004, 1334900005, 1334900006, 1550280001, 1404360004, 1245640005, 1245640006, 1404360005, 1404360006, 1245640009, 1404360007, 1404360009, 1404360010, 1550280010, 1245640014, 1245640015, 1245640016, 1404360016, 1404360018, 1550280018, 269170001, 1550280012, 1404360022, 1404360023, 269170002, 269170009, 269170008, 1550280027, 269170003, 1404360029, 1550280029, 1550280030, 1245640032, 1550280031, 1245640034, 1550280035, 1245640036, 1245640037, 1550280036, 1550280037, 1550280039, 1550280041, 1550280042, 1550280043, 1550280045, 1550280049, 1805510001, 1550280051, 1550280052, 1550280053, 1805510002, 1805510003, 1805510006, 1805510008, 1805510009, 1805510010, 1805510011, 1805510012, 1805510005, 1805510015, 1805510016, 1805510013, 1805510014, 1805510017, 1805510018, 1805510021, 1805510019, 1805510020, 1805510022, 1805510023, 1805510026, 1805510024, 1805510025, 1805510027, 1805510028, 1805510029, 1805510030, 1805510031, 1805510032, 1805510033, 2163140004, 2163140006, 2163140007, 2163140011, 1550280013, 2163140012, 2163140014, 2163140015, 2163140016, 2163140022, 2163140024, 2163140025, 2163140029, 2163140033, 2163140034, 2163140038, 2163140039, 2163140042, 2163140043, 2163140048, 2163140050, 2163140053, 2163140055, 2163140057, 2163140058, 1150310001, 1150310002, 1150310003, 1150310004, 1150310005, 1150310006, 1337530003, 1337530005, 1337530006, 1337530009, 1337530010, 1337530013, 1337530014, 1337530017, 1337530018, 1337530022, 1337530023, 1337530024, 1337530027, 1337530034, 1248270004, 1248270005, 1337530037, 1248270008, 1337530041, 1248270010, 1248270011, 1248270012, 1337530046, 1337530048, 1337530049, 1337530050, 1337530057, 1337530061, 1337530063, 1337530067, 1337530069, 1337530070, 1337530075, 1337530077, 1337530079, 982710005, 982710006, 982710008, 518580001, 518580002, 518580003, 518580004, 518580005, 518580006, 518580010, 518580011, 518580015, 518580016, 518580023, 518580024, 731790005, 731790010, 1488730001, 998660001, 998660002, 998660003, 998660004, 998660005, 2077870001, 2077870002, 2077870003, 2077870004, 2077870005, 720300003, 812460003, 812460004, 812460005, 812460006, 720300008, 812460007, 812460008, 812460009, 720300012, 812460010, 720300017, 1388030001, 269170004, 269170005, 269170006, 269170007, 457550001, 457550002, 457550003, 457550004, 457550005, 1480650002, 1453730001, 1453730002, 1453730003, 1453730004, 1453730005, 1453730006, 1453730007, 1453730008, 1453730009, 1453730010, 1453730011, 1453730012, 1453730013, 1453730014, 1480650007, 600140001, 600140002, 600140003, 600140004, 600140005, 600140006, 600140007, 600140008, 600140009, 600140010, 600140011, 600140012, 600140013, 600140014, 600140015, 600140016, 600140017, 600140018, 600140019, 600140020, 600140021, 600140022, 600140023, 600140024, 866540001, 866540002, 866540003, 866540004, 866540005, 866540006, 866540007, 866540008, 866540009, 866540010, 866540011, 1652800001, 1652800002, 1652800003, 1652800004, 1652800005, 1652800006, 1652800007, 1652800008, 1652800009, 1652800010, 1652800011, 1652800012, 1652800013, 2224020001, 2224020002, 2224020003, 2224020004, 2224020005, 2224020006, 2224020007, 2224020008, 2224020009, 2224020010, 2224020011, 2224020012, 2224020013, 2224020014, 2224020015, 2224020016, 2224020017, 2224020018, 2224020019, 2224020020, 2224020021, 2224020022, 2224020023, 2224020024, 2224020025, 2224020026, 2224020027, 2224020028, 2224020029, 2224020030, 2224020031, 2224020032, 2224020033, 2224020034, 2224020035, 2224020036, 2224020037, 2224020038, 2224020039, 2224020040, 2224020041, 2224020042, 2224020043, 2224020044, 2224020045, 2224020046, 2224020047, 2224020048, 2224020049, 2224020050, 2224020051, 2224020052, 2224020053, 2224020054, 2224020055, 2224020056, 2224020057, 2224020058, 1805510004, 1805510007, 1337060007, 1337060008, 1337060009, 1337060010, 1337060011, 1337060012, 1337060015, 731790001, 1337060018, 1337060019, 1337060020, 1337060021, 1337060022, 1337060023, 1337060024, 1337060025, 1337060026, 731790002, 731790003, 731790004, 1337060030, 731790006, 731790007, 731790008, 731790009, 1337060035, 891960001, 891960002, 891960003, 891960004, 891960005, 891960006, 891960007, 891960008, 891960009, 891960010, 451980001, 451980002, 451980003, 451980004, 451980005, 451980006, 451980007, 451980008, 451980009, 451980010, 451980011, 451980012, 451980013, 451980014, 451980015, 451980016, 451980017, 451980018, 1512840001, 1512840002, 1512840003, 1512840004, 1512840005, 1512840006, 1512840007, 2089520001, 2089520002, 2089520003, 2089520004, 1269890001, 1269890002, 1269890003, 1269890004, 1269890005, 1269890006, 1269890007, 1269890008, 1269890009, 1269890010, 1269890011, 1269890012, 1269890013, 1269890014, 1269890015, 997760001, 997760002, 997760003, 997760004, 997760005, 997760006, 997760007, 997760008, 997760009, 997760010, 997760011, 997760012, 1270910001, 1270910002, 1270910003, 1270910004, 1270910005, 2025000001, 2025000002, 2025000003, 2025000004, 2025000005, 2025000006, 660520005, 2025000007, 2025000009, 2025000010, 660520009, 2025000008, 1862180001, 1862180002, 1862180003, 1862180004, 1862180005, 1862180006, 1862180007, 1157070001, 1382350002, 1157070002, 1382350004, 1157070003, 1382350006, 1382350007, 1382350008, 1157070004, 1157070005, 1157070006, 1157070007, 1157070008, 1931980011, 1264160001, 1264160002, 1264160003, 1264160004, 1264160005, 1264160006, 1264160007, 1264160008, 1264160009, 1264160010, 1264160011, 1264160012, 1264160013, 1264160014, 1264160015, 1264160016, 1480650001, 898080018, 1480650003, 1298890004, 1480650004, 1298890006, 1480650005, 1298890008, 1298890009, 1298890010, 1480650006, 1298890012, 1298890013, 1298890014, 1480650008, 1298890016, 898080026, 898080027, 1298890019, 1298890020, 1298890021, 714100004, 714100006, 714100007, 1026760001, 1026760002, 1026760003, 1026760004, 1026760005, 1026760006, 1026760007, 1026760008, 1026760009, 1026760010, 1026760011, 1026760012, 1026760013, 1026760014, 1026760015, 1026760016, 1026760017, 1026760018, 1026760019, 1026760020, 1026760021, 1026760022, 1026760023, 1026760024, 1026760025, 1026760026, 1026760027, 1113370001, 1210650002, 1113370002, 1113370003, 1210650005, 1113370004, 1210650011, 1210650012, 1210650013, 1210650014, 528580003, 528580004, 1210650023, 1210650024, 1210650025, 1210650026, 1210650027, 1210650028, 1264160017, 758210001, 758210002, 758210003, 758210004, 758210005, 758210006, 758210007, 758210008, 758210009, 758210010, 758210011, 758210012, 758210013, 758210014, 1906880001, 1906880002, 1906880003, 1063530001, 1063530002, 1063530004, 1063530007, 1063530008, 1063530009, 1063530010, 1063530011, 1063530012, 1063530014, 1063530015, 1063530016, 1063530018, 1063530019, 1063530021, 1063530023, 1063530029, 1063530032, 1063530048, 1063530052, 898080019, 1063530057, 1063530059, 1063530065, 1063530067, 1063530069, 1063530070, 898080022, 1063530073, 1063530074, 1063530079, 1063530083, 898080028, 463290002, 463290003, 463290008, 1595490008, 490250001, 490250002, 490250003, 490250004, 490250005, 490250006, 490250007, 490250008, 490250009, 490250010, 490250011, 490250012, 490250013, 490250014, 490250015, 490250016, 490250017, 1825460002, 1825460003, 490250018, 490250019, 1825460001, 1825460004, 1825460005, 1825460009, 1825460006, 1825460007, 1825460008, 1825460010, 1825460011, 1825460012, 815280001, 815280002, 815280003, 815280004, 815280005, 815280006, 815280007, 815280008, 815280009, 815280010, 815280011, 815280012, 815280013, 815280014, 815280015, 815280016, 815280017, 815280018, 2020950001, 2020950002, 2020950003

dataset/split_coco.py

@@ -0,0 +1,166 @@
+import os
+import json
+import shutil
+import argparse
+import random
+import pandas as pd
+
+FOLD_CNT = 10
+
+
+def generate_fold(args):
+    df = pd.DataFrame(columns=['img_id', 'fold'])
+    json_file_path = args.ann_file
+    assert os.path.exists(json_file_path), f'json file not found: {json_file_path}'
+    with open(json_file_path, 'r') as f:
+        json_data = json.load(f)
+    
+    # reproducable random shuffle
+    random.seed(1)
+    random.shuffle(json_data['images'])
+    fold_size = len(json_data['images']) // FOLD_CNT
+    for image in json_data['images']:
+        df.loc[len(df), df.columns] = [image['id'], min(len(df) // fold_size, FOLD_CNT - 1)]
+    df.to_csv(args.fold_file, index=False)
+
+
+def generate_fold_app(args):
+    df = pd.DataFrame(columns=['img_id', 'fold'])
+    json_file_path = args.ann_file
+    assert os.path.exists(json_file_path), f'json file not found: {json_file_path}'
+    with open(json_file_path, 'r') as f:
+        json_data = json.load(f)
+
+    apps = set()
+
+    for image in json_data['images']:
+        appid = str(image['id'])[:-3]
+        apps.add(appid)
+    
+    apps = list(apps)
+    bel_app = {}
+    
+    # reproducable random shuffle
+    apps = sorted(apps)
+    random.seed(1)
+    random.shuffle(apps)
+    fold_size = len(apps) // FOLD_CNT
+    for app in apps:
+        bel_app[app] = min(len(bel_app) // fold_size, FOLD_CNT - 1)
+    for image in json_data['images']:
+        appid = str(image['id'])[:-3]
+        df.loc[len(df), df.columns] = [image['id'], bel_app[appid]]
+    
+    df = df.sort_values(by='fold')
+    df.to_csv(args.fold_file, index=False)
+
+
+def split(args):
+    json_file_path = args.ann_file
+    image_path = args.img_dir if args.img_dir and os.path.exists(args.img_dir) else None
+    fold_file = args.fold_file
+    output_path = args.output_path
+    
+    assert os.path.exists(json_file_path), f'json file not found: {json_file_path}'
+    # assert os.path.exists(image_path), f'image path not found: {image_path}'
+    assert os.path.exists(fold_file), f'fold file not found: {fold_file}'
+
+    with open(json_file_path, 'r') as f:
+        json_data = json.load(f)
+    
+    df = pd.read_csv(fold_file)
+    
+    train_dataset = {'images': list(), 'categories': json_data['categories'], 'annotations': list()}
+    val_dataset = {'images': list(), 'categories': json_data['categories'], 'annotations': list()}
+    test_dataset = {'images': list(), 'categories': json_data['categories'], 'annotations': list()}
+
+    train_folds = [int(fold) for fold in args.train_folds.split(',')] if args.train_folds else []
+    val_folds = [int(fold) for fold in args.val_folds.split(',')] if args.val_folds else []
+    test_folds = [int(fold) for fold in args.test_folds.split(',')] if args.test_folds else []
+
+    print(f'train folds: {train_folds}')
+    print(f'val folds: {val_folds}')
+    print(f'test folds: {test_folds}')
+    
+    train_imgid = set()
+    val_imgid = set()
+    test_imgid = set()
+
+    for _, row in df.iterrows():
+        if row['fold'] in train_folds:
+            train_imgid.add(row['img_id'])
+        elif row['fold'] in val_folds:
+            val_imgid.add(row['img_id'])
+        elif row['fold'] in test_folds:
+            test_imgid.add(row['img_id'])
+        else:
+            raise ValueError(f'fold not found: {row["fold"]}')
+
+    for image in json_data['images']:
+        if image['id'] in train_imgid:
+            train_dataset['images'].append(image)
+        elif image['id'] in val_imgid:
+            val_dataset['images'].append(image)
+        elif image['id'] in test_imgid:
+            test_dataset['images'].append(image)
+        else:
+            raise ValueError(f'image id not found: {image["id"]}')
+ 
+    for annotation in json_data['annotations']:
+        if annotation['image_id'] in train_imgid:
+            train_dataset['annotations'].append(annotation)
+        elif annotation['image_id'] in val_imgid:
+            val_dataset['annotations'].append(annotation)
+        elif annotation['image_id'] in test_imgid:
+            test_dataset['annotations'].append(annotation)
+        else:
+            raise ValueError(f'annotation image_id not found: {annotation["image_id"]}')
+    
+    print(f'train dataset: {len(train_dataset["images"])} images, {len(train_dataset["annotations"])} annotations')
+    print(f'val dataset: {len(val_dataset["images"])} images, {len(val_dataset["annotations"])} annotations')
+    print(f'test dataset: {len(test_dataset["images"])} images, {len(test_dataset["annotations"])} annotations')
+    
+    print(os.path.abspath(output_path))
+    train_image_path = os.path.join(output_path, 'images', 'instances_train2017')
+    val_image_path = os.path.join(output_path, 'images', 'instances_val2017')
+    test_image_path = os.path.join(output_path, 'images', 'instances_test2017')
+    os.makedirs(output_path, exist_ok=True)
+    os.makedirs(train_image_path, exist_ok=True)
+    os.makedirs(val_image_path, exist_ok=True)
+    os.makedirs(test_image_path, exist_ok=True)
+    os.makedirs(os.path.join(output_path, 'annotations'), exist_ok=True)
+    
+    with open(os.path.join(output_path, 'annotations', 'instances_train2017.json'), 'w') as f:
+        json.dump(train_dataset, f, indent=4)
+    with open(os.path.join(output_path, 'annotations', 'instances_val2017.json'), 'w') as f:
+        json.dump(val_dataset, f, indent=4)
+    with open(os.path.join(output_path, 'annotations', 'instances_test2017.json'), 'w') as f:
+        json.dump(test_dataset, f, indent=4)
+
+    if image_path:
+        for image in train_dataset['images']:
+            shutil.copy(os.path.join(image_path, image['file_name']), train_image_path)
+        for image in val_dataset['images']:
+            shutil.copy(os.path.join(image_path, image['file_name']), val_image_path)
+        for image in test_dataset['images']:
+            shutil.copy(os.path.join(image_path, image['file_name']), test_image_path)
+
+
+def main(args):
+    if args.gen:
+       generate_fold_app(args)
+    split(args)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--ann_file', type=str, required=True)
+    parser.add_argument('--img_dir', type=str, required=True)
+    parser.add_argument('--output_path', type=str, default='./coco_split')
+    parser.add_argument('--train_folds', type=str, default=None)
+    parser.add_argument('--val_folds', type=str, default=None)
+    parser.add_argument('--test_folds', type=str, default=None)
+    parser.add_argument('--gen', action='store_true')
+    parser.add_argument('--fold_file', type=str, required=True)
+    args = parser.parse_args()
+    main(args)

environment.reference.yml

@@ -0,0 +1,35 @@
+name: orienter-ape-reference
+channels:
+  - pytorch
+  - nvidia
+  - conda-forge
+dependencies:
+  - python=3.10
+  - pip
+  - pytorch=2.2.0
+  - torchvision=0.17.0
+  - pytorch-cuda=12.1
+  - cuda-version=12.1
+  - numpy=1.22.4
+  - scipy=1.7.3
+  - pillow=12.3.0
+  - pip:
+      - beautifulsoup4==4.12.3
+      - cython==3.0.8
+      - einops==0.7.0
+      - fairscale==0.4.13
+      - fvcore==0.1.5.post20221221
+      - iopath==0.1.9
+      - lvis==0.5.3
+      - omegaconf==2.1.0
+      - opencv-python==4.9.0.80
+      - pycocotools==2.0.7
+      - pyyaml==6.0.1
+      - requests==2.31.0
+      - timm==0.9.12
+      - tqdm==4.66.1
+      - transformers==4.37.2
+      - xformers==0.0.24
+      - git+https://github.com/facebookresearch/detectron2.git@017abbfa5f2c2a2afa045200c2af9ccf2fc6227f
+      - git+https://github.com/IDEA-Research/detrex.git@776058ec229be37a5ff2a2b0bb54133bdd5da663
+      - git+https://github.com/openai/CLIP.git@d50d76daa670286dd6cacf3bcd80b5e4823fc8e1

requirements.txt

@@ -0,0 +1,5 @@
+beautifulsoup4>=4.12,<5
+Pillow>=12.3,<13
+PyYAML>=6,<7
+requests>=2.31,<3
+tqdm>=4.66,<5

tests/test_ape_stage.py

@@ -0,0 +1,176 @@
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO_ROOT))
+
+from approach.ape_stage import extract_image_id, run_ape_stage
+
+
+class ApeStageTests(unittest.TestCase):
+    def test_extract_image_id_requires_documented_basename_shape(self):
+        self.assertEqual(extract_image_id("123_4.jpg"), 123004)
+        with self.assertRaises(ValueError):
+            extract_image_id("123_4_view_a.png")
+        with self.assertRaises(ValueError):
+            extract_image_id("123_view_a.png")
+
+    def test_multiple_records_are_enriched_and_checkpointed_as_valid_json(self):
+        calls = []
+
+        def fake_inference(**kwargs):
+            calls.append(kwargs)
+            category = kwargs["text_prompt"].split(":", 1)[0]
+            return [
+                {
+                    "category_name": category,
+                    "bbox": [1, 2, 3, 4],
+                    "score": 0.9,
+                }
+            ]
+
+        records = [
+            {"question_id": 0, "text": {"objects": {"button": "round red"}}},
+            {"question_id": 1, "text": {"objects": {"lever": "long silver"}}},
+        ]
+        questions = {
+            0: {"image": "123_4.jpg"},
+            1: {"image": "456_78.jpg"},
+        }
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            output_path = root / "predictions.json"
+            results, errors = run_ape_stage(
+                records=records,
+                questions=questions,
+                images_dir=root / "images",
+                output_path=output_path,
+                inference=fake_inference,
+                inference_kwargs={"confidence_threshold": 0.15},
+                resume=False,
+            )
+
+            on_disk = json.loads(output_path.read_text())
+
+        self.assertEqual(errors, [])
+        self.assertEqual(results, on_disk)
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results[0]["image_id"], 123004)
+        self.assertEqual(results[0]["category_id"], "button")
+        self.assertEqual(results[1]["image_id"], 456078)
+        self.assertEqual(results[1]["category_id"], "lever")
+        self.assertEqual(calls[0]["confidence_threshold"], 0.15)
+
+    def test_resume_does_not_duplicate_completed_images(self):
+        calls = []
+
+        def fake_inference(**kwargs):
+            calls.append(kwargs)
+            return [
+                {
+                    "category_name": "button",
+                    "bbox": [1, 2, 3, 4],
+                    "score": 0.9,
+                }
+            ]
+
+        records = [{"question_id": 0, "text": {"objects": {"button": "red"}}}]
+        questions = {0: {"image": "123_4.jpg"}}
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            output_path = root / "predictions.json"
+            first, _ = run_ape_stage(
+                records,
+                questions,
+                root,
+                output_path,
+                fake_inference,
+                resume=False,
+            )
+            second, _ = run_ape_stage(
+                records,
+                questions,
+                root,
+                output_path,
+                fake_inference,
+                resume=True,
+            )
+
+        self.assertEqual(first, second)
+        self.assertEqual(len(calls), 1)
+
+    def test_resume_tracks_completed_images_with_zero_detections(self):
+        calls = []
+
+        def empty_inference(**kwargs):
+            calls.append(kwargs)
+            return []
+
+        records = [{"question_id": "0", "text": {"objects": {"button": "red"}}}]
+        questions = {0: {"image": "123_4.jpg"}}
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            output_path = root / "predictions.json"
+            first, _ = run_ape_stage(
+                records,
+                questions,
+                root,
+                output_path,
+                empty_inference,
+                resume=False,
+            )
+            second, _ = run_ape_stage(
+                records,
+                questions,
+                root,
+                output_path,
+                empty_inference,
+                resume=True,
+            )
+
+            progress = json.loads((root / "predictions.progress.json").read_text())
+
+        self.assertEqual(first, [])
+        self.assertEqual(second, [])
+        self.assertEqual(calls, [calls[0]])
+        self.assertEqual(progress["completed_image_ids"], [123004])
+
+    def test_errors_are_recorded_without_corrupting_predictions(self):
+        def failing_inference(**kwargs):
+            raise RuntimeError("synthetic detector failure")
+
+        records = [{"question_id": 0, "text": {"objects": {"button": "red"}}}]
+        questions = {0: {"image": "123_4.jpg"}}
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            output_path = root / "predictions.json"
+            error_path = root / "errors.json"
+            results, errors = run_ape_stage(
+                records,
+                questions,
+                root,
+                output_path,
+                failing_inference,
+                error_path=error_path,
+                resume=False,
+            )
+
+            self.assertEqual(json.loads(output_path.read_text()), [])
+            self.assertEqual(json.loads(error_path.read_text()), errors)
+
+        self.assertEqual(results, [])
+        self.assertEqual(errors[0]["question_id"], 0)
+        self.assertEqual(errors[0]["error_type"], "RuntimeError")
+        self.assertNotIn("traceback", errors[0])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_app_metadata_cache.py

@@ -0,0 +1,124 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from approach.vlm.gpt4v.gpt4v import get_steam_app_data, load_app_metadata_cache
+
+
+class AppMetadataCacheTests(unittest.TestCase):
+    def test_json_cache_supplies_app_context(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "app_metadata.json"
+            cache_path.write_text(
+                json.dumps(
+                    {
+                        "123": {
+                            "app_name": "Pinned App",
+                            "app_description": "Pinned description",
+                        }
+                    }
+                ),
+                encoding="utf-8",
+            )
+
+            cache = load_app_metadata_cache(cache_path)
+
+        self.assertEqual(
+            get_steam_app_data("123", "123_4.jpg", metadata_cache=cache),
+            ("Pinned App", "Pinned description"),
+        )
+
+    def test_cache_missing_app_id_fails_closed(self):
+        with self.assertRaises(KeyError):
+            get_steam_app_data("999", "999_1.jpg", metadata_cache={})
+
+    def test_jsonl_cache_is_keyed_by_app_id(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "app_metadata.jsonl"
+            cache_path.write_text(
+                json.dumps(
+                    {
+                        "app_id": 456,
+                        "name": "JSONL App",
+                        "description": "JSONL description",
+                    }
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+
+            cache = load_app_metadata_cache(cache_path)
+
+        self.assertEqual(
+            get_steam_app_data("456", "456_7.jpg", metadata_cache=cache),
+            ("JSONL App", "JSONL description"),
+        )
+
+    def test_duplicate_jsonl_app_id_is_rejected(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "app_metadata.jsonl"
+            cache_path.write_text(
+                "\n".join(
+                    [
+                        json.dumps(
+                            {
+                                "app_id": 123,
+                                "app_name": "First",
+                                "app_description": "Description",
+                            }
+                        ),
+                        json.dumps(
+                            {
+                                "app_id": "123",
+                                "app_name": "Second",
+                                "app_description": "Description",
+                            }
+                        ),
+                    ]
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+
+            with self.assertRaises(ValueError):
+                load_app_metadata_cache(cache_path)
+
+    def test_duplicate_json_object_key_is_rejected(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "app_metadata.json"
+            cache_path.write_text(
+                '{"123":{"app_name":"First","app_description":"One"},'
+                '"123":{"app_name":"Second","app_description":"Two"}}',
+                encoding="utf-8",
+            )
+
+            with self.assertRaises(ValueError):
+                load_app_metadata_cache(cache_path)
+
+    def test_mapping_key_and_record_app_id_must_match(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "app_metadata.json"
+            cache_path.write_text(
+                json.dumps(
+                    {
+                        "123": {
+                            "app_id": "456",
+                            "app_name": "Mismatch",
+                            "app_description": "Description",
+                        }
+                    }
+                ),
+                encoding="utf-8",
+            )
+
+            with self.assertRaises(ValueError):
+                load_app_metadata_cache(cache_path)
+
+    def test_missing_cache_fields_fail_closed(self):
+        with self.assertRaises(ValueError):
+            get_steam_app_data("123", "123_4.jpg", metadata_cache={"123": {"app_name": "App"}})
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_cli_entrypoints.py

@@ -0,0 +1,147 @@
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class CliEntrypointTests(unittest.TestCase):
+    def test_run_vlm_script_path_supports_help(self):
+        result = subprocess.run(
+            [sys.executable, str(ROOT / "approach" / "run_vlm.py"), "--help"],
+            cwd=ROOT,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("--questions", result.stdout)
+
+    def test_run_vlm_returns_nonzero_when_a_record_fails(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            questions = root / "questions.jsonl"
+            questions.write_text(
+                json.dumps(
+                    {
+                        "question_id": 0,
+                        "image": "123_4.jpg",
+                        "image_id": 123004,
+                        "text": "mine",
+                    }
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            metadata = root / "metadata.json"
+            metadata.write_text(
+                json.dumps(
+                    {
+                        "456": {
+                            "app_name": "Different app",
+                            "app_description": "Does not cover the selected screenshot.",
+                        }
+                    }
+                ),
+                encoding="utf-8",
+            )
+            result = subprocess.run(
+                [
+                    sys.executable,
+                    "-B",
+                    str(ROOT / "approach" / "run_vlm.py"),
+                    "--questions",
+                    str(questions),
+                    "--images-dir",
+                    str(root / "images"),
+                    "--output",
+                    str(root / "answers.jsonl"),
+                    "--app-metadata-cache",
+                    str(metadata),
+                ],
+                cwd=ROOT,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=False,
+            )
+
+        self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
+        self.assertEqual(json.loads(result.stdout)["errors"], 1)
+
+    def test_run_ape_script_path_supports_help(self):
+        result = subprocess.run(
+            [sys.executable, str(ROOT / "approach" / "run_ape.py"), "--help"],
+            cwd=ROOT,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("--candidates", result.stdout)
+
+    def test_generate_questions_script_path_supports_help(self):
+        result = subprocess.run(
+            [sys.executable, str(ROOT / "scripts" / "generate_questions.py"), "--help"],
+            cwd=ROOT,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("--images-dir", result.stdout)
+
+    def test_environment_check_script_path_supports_help(self):
+        result = subprocess.run(
+            [sys.executable, str(ROOT / "scripts" / "check_environment.py"), "--help"],
+            cwd=ROOT,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("--allow-no-cuda", result.stdout)
+
+    def test_release_test_runner_is_present(self):
+        self.assertTrue((ROOT / "scripts" / "run_tests.py").is_file())
+
+    def test_offline_control_path_smoke(self):
+        result = subprocess.run(
+            [sys.executable, str(ROOT / "scripts" / "smoke_control_path.py")],
+            cwd=ROOT,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn('"status": "ok"', result.stdout)
+
+    def test_ape_build_script_has_valid_shell_syntax(self):
+        result = subprocess.run(
+            ["bash", "-n", str(ROOT / "scripts" / "build_ape_extension.sh")],
+            cwd=ROOT,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_config_providers.py

@@ -0,0 +1,131 @@
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from approach.config import (
+    OPENROUTER_BASE_URL,
+    get_model_profile,
+    sanitize_manifest,
+    validate_model_available,
+)
+from approach.providers import EncodedImage, OpenAICompatibleChatClient, make_multimodal_content
+
+
+class ConfigProviderTests(unittest.TestCase):
+    def test_default_profile_uses_openrouter(self):
+        profile = get_model_profile("default")
+        self.assertEqual(profile.provider, "openrouter")
+        self.assertEqual(profile.model, "openai/gpt-5.6-sol")
+        self.assertEqual(profile.base_url, OPENROUTER_BASE_URL)
+        self.assertEqual(profile.api_key_env, "OPENROUTER_API_KEY")
+
+    def test_aliases_keep_paper_profiles_available(self):
+        self.assertEqual(get_model_profile("gpt4v").model, "openai/gpt-4o-2024-08-06")
+        self.assertEqual(get_model_profile("claude35sonnet").model, "anthropic/claude-3.5-sonnet")
+        self.assertEqual(get_model_profile("gemini15pro").model, "google/gemini-pro-1.5")
+
+    def test_json_config_override_without_yaml_dependency(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = Path(tmpdir) / "profiles.json"
+            path.write_text(
+                json.dumps(
+                    {
+                        "profiles": {
+                            "default": {
+                                "provider": "openrouter",
+                                "model": "custom/model",
+                                "base_url": OPENROUTER_BASE_URL,
+                                "api_key_env": "CUSTOM_KEY",
+                            }
+                        }
+                    }
+                )
+            )
+            profile = get_model_profile("default", str(path))
+        self.assertEqual(profile.model, "custom/model")
+        self.assertEqual(profile.api_key_env, "CUSTOM_KEY")
+
+    def test_model_catalog_validation_is_dependency_injected(self):
+        profile = get_model_profile("default")
+        self.assertTrue(validate_model_available(profile, [{"id": "openai/gpt-5.6-sol"}]))
+        with self.assertRaises(ValueError):
+            validate_model_available(profile, [{"id": "other/model"}])
+
+    def test_multimodal_content_supports_multiple_images(self):
+        content = make_multimodal_content(
+            "check",
+            ["aaa", EncodedImage(data="bbb", media_type="image/png")],
+        )
+        self.assertEqual(content[0], {"type": "text", "text": "check"})
+        self.assertEqual(len(content), 3)
+        self.assertEqual(content[1]["image_url"]["url"], "data:image/jpeg;base64,aaa")
+        self.assertEqual(content[2]["image_url"]["url"], "data:image/png;base64,bbb")
+
+    def test_client_posts_openai_compatible_payload(self):
+        profile = get_model_profile("default")
+        response = mock.Mock()
+        response.json.return_value = {
+            "model": profile.model,
+            "choices": [{"message": {"content": "{\"ok\": true}"}}],
+        }
+        post = mock.Mock(return_value=response)
+
+        with mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-token"}):
+            client = OpenAICompatibleChatClient(profile, post=post)
+            payload = client.build_payload(
+                "verify",
+                ["image-a", "image-b"],
+                response_format={"type": "json_schema", "json_schema": {"name": "x"}},
+            )
+            result = client.complete_json(
+                "verify",
+                ["image-a", "image-b"],
+                response_format={"type": "json_schema", "json_schema": {"name": "x"}},
+            )
+
+        self.assertEqual(payload["model"], "openai/gpt-5.6-sol")
+        self.assertEqual(result, {"ok": True})
+        self.assertEqual(
+            post.call_args.args[0],
+            "https://openrouter.ai/api/v1/chat/completions",
+        )
+        self.assertEqual(post.call_args.kwargs["headers"]["Authorization"], "Bearer test-token")
+        self.assertIn("response_format", post.call_args.kwargs["json"])
+
+    def test_client_supports_multi_turn_json_requests(self):
+        profile = get_model_profile("paper_claude")
+        response = mock.Mock()
+        response.json.return_value = {
+            "choices": [{"message": {"content": '{"objects": ["button"]}'}}]
+        }
+        post = mock.Mock(return_value=response)
+        messages = [
+            {"role": "user", "content": "list objects"},
+            {"role": "assistant", "content": '{"objects": ["button"]}'},
+            {"role": "user", "content": "filter interactable objects"},
+        ]
+
+        with mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-token"}):
+            result = OpenAICompatibleChatClient(profile, post=post).complete_json_messages(
+                messages
+            )
+
+        self.assertEqual(result, {"objects": ["button"]})
+        self.assertEqual(post.call_args.kwargs["json"]["messages"], messages)
+
+    def test_manifest_never_records_api_key_value(self):
+        profile = get_model_profile("default")
+        manifest = sanitize_manifest(
+            profile,
+            {"model": "openai/gpt-5.6-sol"},
+            {"model": "openai/gpt-5.6-sol", "usage": {"total_tokens": 12}},
+        )
+        self.assertEqual(manifest["api_key_env"], "OPENROUTER_API_KEY")
+        self.assertNotIn("sk-", json.dumps(manifest))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_evaluate_temp_paths.py

@@ -0,0 +1,109 @@
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+EVAL_DIR = REPO_ROOT / "evaluation"
+
+PYCOCOTOOLS_STUB = """
+import sys
+import types
+pycocotools = types.ModuleType("pycocotools")
+mask = types.ModuleType("pycocotools._mask")
+def _not_used(*args, **kwargs):
+    raise AssertionError("pycocotools._mask stub should not be exercised in this test")
+for _name in ["iou", "merge", "frPyObjects", "encode", "decode", "area", "toBbox"]:
+    setattr(mask, _name, _not_used)
+pycocotools._mask = mask
+sys.modules["pycocotools"] = pycocotools
+sys.modules["pycocotools._mask"] = mask
+"""
+
+
+def run_eval_import(script: str, env_overrides: dict[str, str]) -> str:
+    env = os.environ.copy()
+    env.update(env_overrides)
+    env.pop("ORIENTER_TMP_ANN_PATH", None)
+    env.pop("ORIENTER_GT_CAT_MATCH_PATH", None)
+    pythonpath = [str(EVAL_DIR), str(REPO_ROOT)]
+    if env.get("PYTHONPATH"):
+        pythonpath.append(env["PYTHONPATH"])
+    env["PYTHONPATH"] = os.pathsep.join(pythonpath)
+    return subprocess.check_output(
+        [sys.executable, "-c", PYCOCOTOOLS_STUB + script],
+        cwd=EVAL_DIR,
+        env=env,
+        text=True,
+    ).strip()
+
+
+class EvaluationTempPathTests(unittest.TestCase):
+    def test_default_temp_paths_are_process_isolated(self):
+        script = """
+import json
+import evaluate_coco
+import pycocotools_ovod.semantic_matching as semantic_matching
+print(json.dumps({
+    "tmp_ann": evaluate_coco.tmp_ann_path,
+    "gt_match": semantic_matching.gt_cat_match_path,
+}))
+"""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            env = {"ORIENTER_EVALUATION_TMPDIR": tmpdir}
+            first = json.loads(run_eval_import(script, env))
+            second = json.loads(run_eval_import(script, env))
+
+        self.assertNotEqual(first["tmp_ann"], second["tmp_ann"])
+        self.assertNotEqual(first["gt_match"], second["gt_match"])
+        self.assertIn("tmp_ann.", Path(first["tmp_ann"]).name)
+        self.assertIn("tmp_gt_cat_match.", Path(first["gt_match"]).name)
+
+    def test_cleanup_removes_configured_temp_files_and_resets_match_cache(self):
+        script = """
+import json
+from pathlib import Path
+import evaluate_coco
+import pycocotools_ovod.semantic_matching as semantic_matching
+
+semantic_matching.gt_cat_match_dict = {"Button": ["Button"]}
+for path in [evaluate_coco.tmp_ann_path, semantic_matching.gt_cat_match_path]:
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    Path(path).write_text("{}")
+
+evaluate_coco.cleanup_temp_outputs()
+print(json.dumps({
+    "tmp_ann_exists": Path(evaluate_coco.tmp_ann_path).exists(),
+    "gt_match_exists": Path(semantic_matching.gt_cat_match_path).exists(),
+    "match_cache": semantic_matching.gt_cat_match_dict,
+}))
+"""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp = Path(tmpdir)
+            env = os.environ.copy()
+            env.update(
+                {
+                    "ORIENTER_TMP_ANN_PATH": str(tmp / "custom_tmp_ann.json"),
+                    "ORIENTER_GT_CAT_MATCH_PATH": str(tmp / "custom_gt_match.json"),
+                    "PYTHONPATH": os.pathsep.join([str(EVAL_DIR), str(REPO_ROOT), env.get("PYTHONPATH", "")]),
+                }
+            )
+            result = subprocess.check_output(
+                [sys.executable, "-c", PYCOCOTOOLS_STUB + script],
+                cwd=EVAL_DIR,
+                env=env,
+                text=True,
+            ).strip()
+
+        state = json.loads(result)
+        self.assertFalse(state["tmp_ann_exists"])
+        self.assertFalse(state["gt_match_exists"])
+        self.assertIsNone(state["match_cache"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_evaluation_config.py

@@ -0,0 +1,26 @@
+import importlib.util
+import unittest
+from pathlib import Path
+
+
+def load_module(path):
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class EvaluationConfigTest(unittest.TestCase):
+    def test_context_baseline_lists_are_deduplicated(self):
+        module = load_module(Path("evaluation/context_eval_baselines.py"))
+        self.assertEqual(len(module.METHODS), len(set(module.METHODS)))
+        self.assertEqual(len(module.LLM_METHODS), len(set(module.LLM_METHODS)))
+
+    def test_to_pred_requires_explicit_task(self):
+        module = load_module(Path("evaluation/tools/to_pred.py"))
+        self.assertIn("interactable", module.TASKS)
+        self.assertIn("all", module.build_parser()._option_string_actions["--task"].choices)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_evaluation_tools.py

@@ -0,0 +1,145 @@
+import json
+import sys
+import tempfile
+import unittest
+from itertools import chain, repeat
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from evaluation import context_eval_baselines
+from evaluation.tools import to_pred
+
+
+class ToPredTests(unittest.TestCase):
+    def test_task_is_required(self):
+        parser = to_pred.build_parser()
+
+        with self.assertRaises(SystemExit):
+            parser.parse_args(["--input", "pred.json", "--output", "out.json"])
+
+    def test_convert_old_predictions_to_semantics_fields(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp = Path(tmpdir)
+            pred_path = tmp / "pred.json"
+            question_path = tmp / "questions.jsonl"
+            output_path = tmp / "out.json"
+
+            pred_path.write_text(
+                json.dumps(
+                    {
+                        "q1": {
+                            "oovd_result": {
+                                "button": [
+                                    {
+                                        "bbox_pixels": [1, 2, 3, 4],
+                                        "probability": 0.7,
+                                    }
+                                ]
+                            }
+                        }
+                    }
+                )
+            )
+            question_path.write_text(
+                json.dumps({"question_id": "q1", "image": "1026760_11.jpg"}) + "\n"
+            )
+
+            to_pred.main(
+                [
+                    "--task",
+                    "semantics",
+                    "--input",
+                    str(pred_path),
+                    "--questions",
+                    str(question_path),
+                    "--output",
+                    str(output_path),
+                ]
+            )
+
+            converted = json.loads(output_path.read_text())
+            self.assertEqual(
+                converted,
+                [
+                    {
+                        "image_id": 1026760011,
+                        "category_id": "button",
+                        "bbox": [1, 2, 3, 4],
+                        "score": 0.7,
+                    }
+                ],
+            )
+
+    def test_all_outputs_do_not_overwrite(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp = Path(tmpdir)
+            pred_path = tmp / "pred.json"
+            output_path = tmp / "converted.json"
+
+            pred_path.write_text(
+                json.dumps(
+                    [
+                        {
+                            "image_id": 1,
+                            "category_id": "trigger",
+                            "bbox": [1, 2, 3, 4],
+                            "score": 0.5,
+                        }
+                    ]
+                )
+            )
+
+            to_pred.main(
+                [
+                    "--task",
+                    "all",
+                    "--input",
+                    str(pred_path),
+                    "--output",
+                    str(output_path),
+                ]
+            )
+
+            self.assertFalse(output_path.exists())
+            for task in to_pred.TASKS:
+                self.assertTrue((tmp / f"converted_{task}.json").exists())
+
+            interactable = json.loads((tmp / "converted_interactable.json").read_text())
+            interaction = json.loads((tmp / "converted_interaction.json").read_text())
+            semantics = json.loads((tmp / "converted_semantics.json").read_text())
+            self.assertEqual(interactable[0]["category_id"], 1)
+            self.assertEqual(interaction[0]["category_id"], "trigger")
+            self.assertEqual(semantics[0]["category_id"], "trigger")
+
+
+class ContextEvalBaselineTests(unittest.TestCase):
+    def test_validate_unique_methods_rejects_duplicates(self):
+        with self.assertRaises(ValueError):
+            context_eval_baselines.validate_unique_methods(["Seed-E2E", "Seed-E2E"])
+
+    def test_main_uses_subprocess_check_true(self):
+        args = context_eval_baselines.build_parser().parse_args([])
+        methods = ["CenterNet2"]
+
+        with mock.patch.object(
+            context_eval_baselines, "METHODS", methods
+        ), mock.patch.object(
+            context_eval_baselines, "LLM_METHODS", []
+        ), mock.patch.object(
+            context_eval_baselines.os.path,
+            "exists",
+            side_effect=chain([True, False], repeat(False)),
+        ), mock.patch.object(
+            context_eval_baselines.subprocess, "run"
+        ) as run:
+            context_eval_baselines.main(args)
+
+        self.assertTrue(run.called)
+        self.assertTrue(run.call_args.kwargs["check"])
+        self.assertIsInstance(run.call_args.args[0], list)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_merge_predictions.py

@@ -0,0 +1,62 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from evaluation.tools.merge_predictions import merge_prediction_shards
+
+
+class MergePredictionTests(unittest.TestCase):
+    def test_merges_disjoint_shards_and_validates_question_coverage(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            first = root / "pred.shard00-of-02.json"
+            second = root / "pred.shard01-of-02.json"
+            output = root / "merged.json"
+            first.write_text(json.dumps([{"image_id": 1001, "category_id": "button", "bbox": [1, 2, 3, 4], "score": 0.9}]))
+            second.write_text(json.dumps([{"image_id": 1002, "category_id": "lever", "bbox": [4, 3, 2, 1], "score": 0.8}]))
+            first.with_suffix(".progress.json").write_text(json.dumps({"completed_image_ids": [1001]}))
+            second.with_suffix(".progress.json").write_text(json.dumps({"completed_image_ids": [1002]}))
+
+            report = merge_prediction_shards(
+                [first, second],
+                output,
+                expected_image_ids={1001, 1002},
+            )
+
+            self.assertEqual(len(json.loads(output.read_text())), 2)
+            self.assertEqual(report["completed_image_ids"], 2)
+            self.assertEqual(report["missing_image_ids"], [])
+
+    def test_rejects_overlapping_shard_progress(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            paths = [root / "a.json", root / "b.json"]
+            for path in paths:
+                path.write_text("[]")
+                path.with_suffix(".progress.json").write_text(
+                    json.dumps({"completed_image_ids": [1001]})
+                )
+
+            with self.assertRaisesRegex(ValueError, "overlap"):
+                merge_prediction_shards(paths, root / "merged.json")
+
+    def test_rejects_missing_expected_images(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            shard = root / "a.json"
+            shard.write_text("[]")
+            shard.with_suffix(".progress.json").write_text(
+                json.dumps({"completed_image_ids": [1001]})
+            )
+
+            with self.assertRaisesRegex(ValueError, "missing"):
+                merge_prediction_shards(
+                    [shard],
+                    root / "merged.json",
+                    expected_image_ids={1001, 1002},
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_paired_bootstrap.py

@@ -0,0 +1,233 @@
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from evaluation.tools import paired_bootstrap
+
+
+def _write(path, payload):
+    path.write_text(json.dumps(payload), encoding="utf-8")
+
+
+class PairedBootstrapTests(unittest.TestCase):
+    def _toy_gt(self):
+        return {
+            "images": [{"id": 1}, {"id": 2}, {"id": 3}],
+            "categories": [
+                {"id": 1, "name": "Button"},
+                {"id": 2, "name": "Sphere"},
+            ],
+            "annotations": [
+                {"id": 1, "image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10]},
+                {"id": 2, "image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10]},
+            ],
+        }
+
+    def test_greedy_matching_uses_score_order_and_complete_images(self):
+        gt = self._toy_gt()
+        preds = [
+            {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.9},
+            {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.2},
+            {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.8},
+        ]
+
+        prepared_gt = paired_bootstrap.prepare_ground_truth(gt)
+        evaluated = paired_bootstrap.evaluate_predictions(
+            prepared_gt,
+            preds,
+            dimension="interactable",
+            iou_threshold=0.75,
+            score_threshold=0.0,
+        )
+
+        self.assertEqual(evaluated.counts["tp"], 2)
+        self.assertEqual(evaluated.counts["fp"], 1)
+        self.assertEqual(evaluated.counts["fn"], 0)
+        self.assertEqual(len(evaluated.per_image), 3)
+        self.assertEqual(evaluated.per_image[3].support, 0)
+
+    def test_auto_threshold_maximizes_micro_f1(self):
+        gt = self._toy_gt()
+        preds = [
+            {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.4},
+            {"image_id": 2, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.9},
+            {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.8},
+            {"image_id": 3, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.4},
+            {"image_id": 3, "category_id": 1, "bbox": [1, 1, 10, 10], "score": 0.4},
+            {"image_id": 3, "category_id": 2, "bbox": [2, 2, 10, 10], "score": 0.4},
+            {"image_id": 3, "category_id": 2, "bbox": [3, 3, 10, 10], "score": 0.4},
+        ]
+
+        prepared_gt = paired_bootstrap.prepare_ground_truth(gt)
+        threshold, evaluated = paired_bootstrap.select_best_threshold(
+            prepared_gt,
+            preds,
+            dimension="interactable",
+            iou_threshold=0.75,
+        )
+
+        self.assertEqual(threshold, 0.8)
+        self.assertAlmostEqual(evaluated.metrics["f1"], 0.5)
+
+    def test_fast_auto_threshold_matches_bruteforce_selection(self):
+        gt = self._toy_gt()
+        preds = [
+            {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.91},
+            {"image_id": 1, "category_id": 1, "bbox": [1, 1, 10, 10], "score": 0.42},
+            {"image_id": 2, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 0.88},
+            {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.73},
+            {"image_id": 3, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.11},
+            {"image_id": 99, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.72},
+        ]
+        prepared_gt = paired_bootstrap.prepare_ground_truth(gt)
+
+        fast_threshold = paired_bootstrap.select_best_threshold_fast(
+            prepared_gt,
+            preds,
+            dimension="interactable",
+            iou_threshold=0.75,
+        )
+        brute_threshold, brute_result = self._select_best_threshold_bruteforce(prepared_gt, preds)
+
+        self.assertEqual(fast_threshold, brute_threshold)
+        self.assertEqual(fast_threshold, 0.72)
+        fast_result = paired_bootstrap.evaluate_predictions(
+            prepared_gt,
+            preds,
+            dimension="interactable",
+            iou_threshold=0.75,
+            score_threshold=fast_threshold,
+        )
+        self.assertEqual(fast_result.metrics, brute_result.metrics)
+
+    def _select_best_threshold_bruteforce(self, prepared_gt, preds):
+        best_threshold = None
+        best_result = None
+        for threshold in sorted({float(pred["score"]) for pred in preds}, reverse=True):
+            result = paired_bootstrap.evaluate_predictions(
+                prepared_gt,
+                preds,
+                dimension="interactable",
+                iou_threshold=0.75,
+                score_threshold=threshold,
+            )
+            if best_result is None or (
+                result.metrics["f1"],
+                result.metrics["precision"],
+                result.metrics["recall"],
+                -threshold,
+            ) > (
+                best_result.metrics["f1"],
+                best_result.metrics["precision"],
+                best_result.metrics["recall"],
+                -best_threshold,
+            ):
+                best_threshold = threshold
+                best_result = result
+        return best_threshold, best_result
+
+    def test_semantic_cache_match_maps_numeric_categories(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp = Path(tmpdir)
+            cache = tmp / "embedding.json"
+            _write(cache, {"Button": [1.0, 0.0], "Control": [0.9, 0.1]})
+            matcher = paired_bootstrap.SemanticMatcher(cache)
+
+            gt = self._toy_gt()
+            preds = [
+                {"image_id": 1, "category_id": "control", "bbox": [0, 0, 10, 10], "score": 1.0}
+            ]
+            prepared_gt = paired_bootstrap.prepare_ground_truth(gt)
+            evaluated = paired_bootstrap.evaluate_predictions(
+                prepared_gt,
+                preds,
+                dimension="semantics",
+                iou_threshold=0.75,
+                score_threshold=0.0,
+                semantic_matcher=matcher,
+            )
+
+            self.assertEqual(evaluated.counts["tp"], 1)
+
+    def test_semantic_cache_preserves_historical_camelcase_keys(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache = Path(tmpdir) / "embedding.json"
+            _write(cache, {"QuitButton": [1.0, 0.0], "ConfirmButton": [0.9, 0.1]})
+            matcher = paired_bootstrap.SemanticMatcher(cache)
+
+            self.assertTrue(matcher.matches("quit_button", "confirm_button"))
+
+    def test_semantic_match_defaults_to_historical_raw_dot_product(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache = Path(tmpdir) / "embedding.json"
+            _write(cache, {"A": [1.0, 0.0], "B": [0.84, 0.20]})
+
+            historical = paired_bootstrap.SemanticMatcher(cache)
+            cosine = paired_bootstrap.SemanticMatcher(cache, similarity_mode="cosine")
+
+            self.assertFalse(historical.matches("a", "b"))
+            self.assertTrue(cosine.matches("a", "b"))
+
+    def test_semantic_cache_miss_is_hard_error(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache = Path(tmpdir) / "embedding.json"
+            _write(cache, {"Button": [1.0, 0.0]})
+            matcher = paired_bootstrap.SemanticMatcher(cache)
+
+            with self.assertRaisesRegex(KeyError, "Missing frozen embedding cache entries"):
+                matcher.matches("button", "unknown")
+
+    def test_cli_writes_json_report(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp = Path(tmpdir)
+            gt_path = tmp / "gt.json"
+            a_path = tmp / "a.json"
+            b_path = tmp / "b.json"
+            out_path = tmp / "report.json"
+            _write(gt_path, self._toy_gt())
+            _write(a_path, [{"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 1.0}])
+            _write(
+                b_path,
+                [
+                    {"image_id": 1, "category_id": 1, "bbox": [0, 0, 10, 10], "score": 1.0},
+                    {"image_id": 2, "category_id": 2, "bbox": [0, 0, 10, 10], "score": 0.9},
+                ],
+            )
+
+            paired_bootstrap.main(
+                [
+                    "--gt",
+                    str(gt_path),
+                    "--method-a",
+                    str(a_path),
+                    "--method-b",
+                    str(b_path),
+                    "--dimension",
+                    "interactable",
+                    "--auto-threshold",
+                    "--replicates",
+                    "100",
+                    "--seed",
+                    "7",
+                    "--output",
+                    str(out_path),
+                ]
+            )
+
+            report = json.loads(out_path.read_text(encoding="utf-8"))
+            self.assertEqual(report["protocol"]["dimension"], "interactable")
+            self.assertEqual(report["protocol"]["bootstrap_unit"], "image_id")
+            self.assertGreater(report["methods"]["method_b"]["point"]["micro"]["f1"], report["methods"]["method_a"]["point"]["micro"]["f1"])
+            self.assertIn("mean_per_all_images", report["methods"]["method_a"]["point"])
+            self.assertIn("mean_per_positive_support_images", report["methods"]["method_a"]["point"])
+            self.assertIn("mean_per_all_images_delta", report["bootstrap"])
+            self.assertIn("mean_per_positive_support_images_delta", report["bootstrap"])
+            self.assertEqual(report["bootstrap"]["delta_direction"], "method_b_minus_method_a")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_pipeline_utils.py

@@ -0,0 +1,123 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from PIL import Image
+
+from approach.pipeline_utils import (
+    enrich_ape_results,
+    output_path_for_selection,
+    parse_orienter_image_name,
+    resolve_image_path,
+    run_optional_reflection,
+    select_jsonl_lines,
+    write_json_atomic,
+)
+
+
+class PipelineUtilsTest(unittest.TestCase):
+    def test_parse_orienter_image_name_is_strict(self):
+        self.assertEqual(parse_orienter_image_name("123_4.jpg"), ("123", "4", 123004))
+        with self.assertRaises(ValueError):
+            parse_orienter_image_name("123_4_extra.jpg")
+        with self.assertRaises(ValueError):
+            parse_orienter_image_name("nested/123_4.jpg")
+        with self.assertRaises(ValueError):
+            parse_orienter_image_name("123_1000.jpg")
+
+    def test_resolve_image_path_rejects_escape_from_images_directory(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            images_dir = Path(tmp_dir) / "images"
+            images_dir.mkdir()
+            self.assertEqual(
+                resolve_image_path(images_dir, "nested/123_4.jpg"),
+                (images_dir / "nested" / "123_4.jpg").resolve(),
+            )
+            with self.assertRaises(ValueError):
+                resolve_image_path(images_dir, "../123_4.jpg")
+            with self.assertRaises(ValueError):
+                resolve_image_path(images_dir, str(Path(tmp_dir) / "123_4.jpg"))
+
+    def test_select_jsonl_lines_supports_ranges_or_shards(self):
+        lines = [f"{idx}\n" for idx in range(10)]
+        self.assertEqual(select_jsonl_lines(lines, start_index=2, end_index=8), lines[2:8])
+        self.assertEqual(
+            select_jsonl_lines(lines, shard_index=1, num_shards=2),
+            ["1\n", "3\n", "5\n", "7\n", "9\n"],
+        )
+        with self.assertRaises(ValueError):
+            select_jsonl_lines(
+                lines,
+                start_index=2,
+                end_index=8,
+                shard_index=1,
+                num_shards=2,
+            )
+
+    def test_output_path_for_selection_prevents_shard_overwrite(self):
+        self.assertEqual(
+            output_path_for_selection("predictions.json", shard_index=1, num_shards=4),
+            "predictions.shard01-of-04.json",
+        )
+        self.assertEqual(
+            output_path_for_selection("predictions.json", start_index=100, end_index=200),
+            "predictions.rows100-200.json",
+        )
+        self.assertEqual(output_path_for_selection("predictions.json"), "predictions.json")
+
+    def test_enrich_ape_results_keeps_added_metadata(self):
+        enriched = enrich_ape_results(
+            [{"bbox": [1, 2, 3, 4], "category_name": "button"}],
+            "123_4.jpg",
+            lambda image_name: 123004,
+        )
+        self.assertEqual(enriched[0]["image_id"], 123004)
+        self.assertEqual(enriched[0]["category_id"], "button")
+        self.assertEqual(enriched[0]["bbox"], [1, 2, 3, 4])
+
+    def test_write_json_atomic_writes_one_valid_json_document(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            out_path = Path(tmp_dir) / "ape.json"
+            write_json_atomic(str(out_path), [{"image_id": 1}, {"image_id": 2}])
+            self.assertEqual(json.loads(out_path.read_text()), [{"image_id": 1}, {"image_id": 2}])
+
+    def test_optional_reflection_is_disabled_by_default(self):
+        self.assertIsNone(
+            run_optional_reflection(
+                "unused.png",
+                [],
+                detector=lambda candidates, previous: previous,
+                enabled=False,
+            )
+        )
+
+    def test_optional_reflection_feeds_back_to_detector(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            image_path = Path(tmp_dir) / "image.png"
+            Image.new("RGB", (10, 10), "white").save(image_path)
+            detector_calls = []
+
+            def advisor(**kwargs):
+                if detector_calls:
+                    return {"verified": [0], "needs_refinement": [], "feedback": []}
+                return {"verified": [], "needs_refinement": [0], "feedback": ["button label"]}
+
+            def detector(candidates, previous):
+                detector_calls.append(candidates)
+                return [{"bbox": [2, 2, 3, 3], "category_name": "button"}]
+
+            result = run_optional_reflection(
+                str(image_path),
+                [{"bbox": [1, 1, 3, 3], "category_name": "button"}],
+                detector=detector,
+                enabled=True,
+                advisor=advisor,
+            )
+
+        self.assertEqual(detector_calls, [["button label"]])
+        self.assertFalse(result["max_iterations_reached"])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_reflection.py

@@ -0,0 +1,147 @@
+import unittest
+
+from PIL import Image
+
+from unittest import mock
+
+from approach.reflection import (
+    build_reflection_prompt,
+    make_successful_detection_request,
+    openai_compatible_advisor,
+    run_reflection_loop,
+)
+
+
+class ReflectionTest(unittest.TestCase):
+    def test_pii5_crop_uses_bbox_without_resize_or_flip(self):
+        image = Image.new("RGB", (12, 8), "black")
+        for x in range(2, 7):
+            for y in range(1, 5):
+                image.putpixel((x, y), (x, y, 100))
+
+        request = make_successful_detection_request(
+            image, {"bbox": [2, 1, 5, 4], "category_name": "button"}
+        )
+
+        self.assertEqual(request.crop_image.size, (5, 4))
+        self.assertEqual(request.crop_image.getpixel((0, 0)), image.getpixel((2, 1)))
+        self.assertEqual(request.crop_image.getpixel((4, 0)), image.getpixel((6, 1)))
+        self.assertIs(request.original_image, image)
+
+    def test_fractional_bbox_crop_contains_the_full_detected_region(self):
+        request = make_successful_detection_request(
+            Image.new("RGB", (10, 10), "white"),
+            {"bbox": [1.2, 2.2, 2.2, 1.2], "category_name": "button"},
+        )
+
+        self.assertEqual(request.crop_image.size, (3, 2))
+
+    def test_reflection_loop_stops_when_advisor_has_no_concerns(self):
+        calls = []
+
+        def advisor(**kwargs):
+            calls.append(kwargs)
+            return {"verified": [0], "needs_refinement": [], "feedback": []}
+
+        result = run_reflection_loop(
+            Image.new("RGB", (10, 10), "white"),
+            [{"bbox": [1, 1, 3, 3], "category_name": "menu"}],
+            miner=lambda trace: ["menu"],
+            detector=lambda candidates, previous: previous,
+            advisor=advisor,
+            max_iterations=10,
+        )
+
+        self.assertEqual(len(calls), 1)
+        self.assertFalse(result["max_iterations_reached"])
+        self.assertEqual(result["trace"][0]["verified"], [0])
+        self.assertEqual(len(calls[0]["verification_requests"]), 1)
+        self.assertIn("PII.6", calls[0]["pii6_prompt"])
+
+    def test_reflection_loop_marks_max_iterations(self):
+        def advisor(**kwargs):
+            return {"verified": [], "needs_refinement": [0], "feedback": ["retry"]}
+
+        result = run_reflection_loop(
+            Image.new("RGB", (10, 10), "white"),
+            [{"bbox": [1, 1, 3, 3], "category_name": "menu"}],
+            miner=lambda trace: ["menu"],
+            detector=lambda candidates, previous: previous,
+            advisor=advisor,
+            max_iterations=3,
+        )
+
+        self.assertTrue(result["max_iterations_reached"])
+        self.assertEqual(len(result["trace"]), 3)
+
+    def test_reflection_removes_rejected_false_positive_before_redetection(self):
+        detector_calls = []
+
+        def detector(candidates, retained):
+            detector_calls.append((candidates, retained))
+            return retained
+
+        advice = [
+            {"verified": [1], "needs_refinement": [0], "feedback": ["missing lever"]},
+            {"verified": [0], "needs_refinement": [], "feedback": []},
+        ]
+        detections = [
+            {"bbox": [1, 1, 2, 2], "category_name": "decoration"},
+            {"bbox": [5, 5, 2, 2], "category_name": "button"},
+        ]
+
+        result = run_reflection_loop(
+            Image.new("RGB", (10, 10), "white"),
+            detections,
+            miner=lambda trace: trace[-1]["feedback"],
+            detector=detector,
+            advisor=lambda **kwargs: advice.pop(0),
+        )
+
+        self.assertEqual(detector_calls[0][0], ["missing lever"])
+        self.assertEqual(detector_calls[0][1], [detections[1]])
+        self.assertEqual(result["detections"], [detections[1]])
+
+    def test_openai_compatible_advisor_sends_annotated_scene_original_and_crop(self):
+        fake_client = mock.Mock()
+        fake_client.complete_json.return_value = {
+            "verified": [0],
+            "needs_refinement": [],
+            "feedback": [],
+        }
+
+        with mock.patch("approach.reflection.OpenAICompatibleChatClient", return_value=fake_client):
+            advisor = openai_compatible_advisor("default")
+            request = make_successful_detection_request(
+                Image.new("RGB", (10, 10), "white"),
+                {"bbox": [1, 1, 3, 3], "category_name": "button"},
+            )
+            result = advisor(
+                verification_requests=[request],
+                unsuccessful_detection_image=Image.new("RGB", (10, 10), "white"),
+                pii5_prompt="PII.5",
+                pii6_prompt="PII.6",
+                pii7_prompt="PII.7",
+            )
+
+        self.assertEqual(result["verified"], [0])
+        call = fake_client.complete_json.call_args
+        self.assertIn("PII.5", call.args[0])
+        self.assertEqual(len(call.args[1]), 3)
+        self.assertTrue(all(image.media_type == "image/png" for image in call.args[1]))
+        self.assertIn("response_format", call.kwargs)
+
+    def test_reflection_prompt_includes_detection_metadata(self):
+        request = make_successful_detection_request(
+            Image.new("RGB", (10, 10), "white"),
+            {"bbox": [1, 2, 3, 4], "category_name": "slider", "score": 0.9},
+        )
+
+        prompt = build_reflection_prompt([request], "PII.5", "PII.6", "PII.7")
+
+        self.assertIn('"category_name": "slider"', prompt)
+        self.assertIn("PII.6", prompt)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_release_hygiene.py

@@ -0,0 +1,151 @@
+import re
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class ReleaseHygieneTests(unittest.TestCase):
+    allowed_large_files = {
+        Path("approach/ovod/APE/ape_d_model_final.pth"),
+    }
+
+    def test_release_policies_exclude_generated_python_state(self):
+        gitignore = (ROOT / ".gitignore").read_text(encoding="utf-8")
+        release_excludes = (ROOT / "docs" / "RELEASE_EXCLUDES.txt").read_text(encoding="utf-8")
+
+        self.assertIn("__pycache__/", gitignore)
+        self.assertIn("*.py[cod]", gitignore)
+        self.assertIn(".pytest_cache/", gitignore)
+        self.assertIn("**/__pycache__/", release_excludes)
+        self.assertIn("**/*.pyc", release_excludes)
+        self.assertIn("**/.pytest_cache/", release_excludes)
+
+    def test_release_does_not_reference_private_relays_or_source_machines(self):
+        forbidden = (
+            "chat" + "anywhere",
+            "aigpt" + "x.top",
+            "/home/" + "sqli/",
+            "/research/d4/gds/" + "sqli21",
+            "10.249." + "190.53",
+            "/Users" + "/",
+            "/users/" + "prannay",
+        )
+        violations = []
+        for path in ROOT.rglob("*"):
+            if not path.is_file() or ".git" in path.parts:
+                continue
+            try:
+                text = path.read_text(encoding="utf-8")
+            except (UnicodeDecodeError, OSError):
+                continue
+            for needle in forbidden:
+                if needle in text:
+                    violations.append(f"{path.relative_to(ROOT)}: {needle}")
+        self.assertEqual(violations, [])
+
+    def test_release_excludes_generated_results_and_local_git_state(self):
+        forbidden_parts = {
+            ".git",
+            "__pycache__",
+            ".pytest_cache",
+            "outputs",
+            "output",
+            "runs",
+            "wandb",
+            "results",
+            "eval_results",
+            "validation",
+        }
+        violations = []
+        for path in ROOT.rglob("*"):
+            rel = path.relative_to(ROOT)
+            if any(part in forbidden_parts for part in rel.parts):
+                violations.append(str(rel))
+        self.assertEqual(violations, [])
+
+        self.assertFalse(
+            (ROOT / "data").exists(),
+            "Top-level data/ must remain an external mount, not release content",
+        )
+
+        explicitly_forbidden = (
+            Path("docs/EVALUATION_VALIDATION.md"),
+            Path("evaluation/validation"),
+            Path("approach/ovod/APE/test_output.png"),
+            Path("baselines/claude-4.5-sonnet-e2e/claude-4.5-sonnet-tiny.json"),
+        )
+        self.assertEqual([str(path) for path in explicitly_forbidden if (ROOT / path).exists()], [])
+
+        forbidden_suffixes = (".log", ".pyc", ".pyo", ".nfs")
+        unexpected = []
+        for path in ROOT.rglob("*"):
+            if not path.is_file():
+                continue
+            if path.name.startswith(".nfs") or path.name.endswith(forbidden_suffixes):
+                unexpected.append(str(path.relative_to(ROOT)))
+        self.assertEqual(unexpected, [])
+
+    def test_release_has_no_credential_material(self):
+        forbidden_files = []
+        for path in ROOT.rglob("*"):
+            if not path.is_file():
+                continue
+            rel = path.relative_to(ROOT)
+            if path.name == ".env" or (
+                path.name.startswith(".env.") and path.name != ".env.example"
+            ):
+                forbidden_files.append(str(rel))
+            if path.suffix.lower() in {".pem", ".key"}:
+                forbidden_files.append(str(rel))
+        self.assertEqual(forbidden_files, [])
+
+        secret_patterns = (
+            re.compile(r"sk-or-v1-[A-Za-z0-9_-]{20,}"),
+            re.compile(r"olp_[A-Za-z0-9_-]{12,}"),
+            re.compile(r"AIza[0-9A-Za-z_-]{20,}"),
+            re.compile(r"gh[pousr]_[0-9A-Za-z]{20,}"),
+            re.compile(r"hf_[0-9A-Za-z]{20,}"),
+            re.compile(r"AKIA[0-9A-Z]{16}"),
+            re.compile(r"BEGIN (?:RSA |OPENSSH |EC )?PRIVATE KEY"),
+        )
+        violations = []
+        for path in ROOT.rglob("*"):
+            if not path.is_file() or path.suffix.lower() == ".pth":
+                continue
+            try:
+                text = path.read_text(encoding="utf-8")
+            except (UnicodeDecodeError, OSError):
+                continue
+            if any(pattern.search(text) for pattern in secret_patterns):
+                violations.append(str(path.relative_to(ROOT)))
+        self.assertEqual(violations, [])
+
+    def test_large_binary_files_are_limited_to_bundled_main_checkpoint(self):
+        threshold = 20 * 1024 * 1024
+        violations = []
+        for path in ROOT.rglob("*"):
+            if not path.is_file():
+                continue
+            rel = path.relative_to(ROOT)
+            if path.stat().st_size > threshold and rel not in self.allowed_large_files:
+                violations.append(str(rel))
+        self.assertEqual(violations, [])
+
+    def test_model_manifest_matches_bundled_checkpoint_policy(self):
+        manifest = (ROOT / "docs" / "MODEL_MANIFEST.md").read_text(encoding="utf-8")
+        checkpoint = ROOT / "approach" / "ovod" / "APE" / "ape_d_model_final.pth"
+        self.assertIn("3548f41a3238148180e08fd4b16c71f4abc3ac3caf9c8434444462d1bdb7f965", manifest)
+        self.assertTrue(checkpoint.is_file())
+        self.assertEqual(checkpoint.stat().st_size, 5_956_547_279)
+
+        attributes = (ROOT / ".gitattributes").read_text(encoding="utf-8")
+        self.assertIn(
+            "approach/ovod/APE/ape_d_model_final.pth filter=lfs diff=lfs merge=lfs -text",
+            attributes,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_release_scripts.py

@@ -0,0 +1,153 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from scripts.generate_questions import build_questions, write_metadata_template
+from scripts.verify_assets import (
+    check_embedding_cache,
+    check_metadata_cache,
+    check_questions,
+    sha256_file,
+)
+
+
+class ReleaseScriptTests(unittest.TestCase):
+    def test_generate_questions_from_image_directory(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            images = root / "images"
+            images.mkdir()
+            (images / "123_4.jpg").write_bytes(b"fake")
+            (images / "123_5.png").write_bytes(b"fake")
+            (images / "123_10.png").write_bytes(b"fake")
+            (images / "notes.txt").write_text("ignored", encoding="utf-8")
+
+            questions = build_questions(images, "Prompt")
+            metadata_template = root / "metadata.json"
+            write_metadata_template(metadata_template, questions)
+
+            metadata = json.loads(metadata_template.read_text(encoding="utf-8"))
+
+        self.assertEqual([record["image_id"] for record in questions], [123004, 123005, 123010])
+        self.assertEqual(questions[0]["text"], "Prompt")
+        self.assertEqual(sorted(metadata), ["123"])
+
+    def test_recursive_question_generation_uses_basename_for_app_id(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            images = root / "images"
+            nested = images / "nested"
+            nested.mkdir(parents=True)
+            (nested / "123_4.jpg").write_bytes(b"fake")
+
+            questions = build_questions(images, "Prompt", recursive=True)
+            metadata_template = root / "metadata.json"
+            write_metadata_template(metadata_template, questions)
+            metadata = json.loads(metadata_template.read_text(encoding="utf-8"))
+
+        self.assertEqual(questions[0]["image"], "nested/123_4.jpg")
+        self.assertEqual(sorted(metadata), ["123"])
+
+    def test_verify_questions_and_metadata_cache(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            images = root / "images"
+            images.mkdir()
+            (images / "123_4.jpg").write_bytes(b"fake")
+            questions = root / "questions.jsonl"
+            questions.write_text(
+                json.dumps(
+                    {
+                        "question_id": 0,
+                        "image": "123_4.jpg",
+                        "image_id": 123004,
+                        "text": "Prompt",
+                    }
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            metadata = root / "metadata.json"
+            metadata.write_text(
+                json.dumps(
+                    {
+                        "123": {
+                            "app_name": "Test VR",
+                            "app_description": "Description.",
+                        }
+                    }
+                ),
+                encoding="utf-8",
+            )
+
+            report = check_questions(questions, images)
+            metadata_report = check_metadata_cache(metadata, report["app_ids"])
+
+        self.assertEqual(report["questions"], 1)
+        self.assertEqual(metadata_report["covered_app_ids"], 1)
+
+    def test_verify_metadata_cache_rejects_duplicates_and_missing_fields(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            duplicate = root / "duplicate.jsonl"
+            duplicate.write_text(
+                "\n".join(
+                    [
+                        json.dumps(
+                            {
+                                "app_id": 123,
+                                "app_name": "First",
+                                "app_description": "Description.",
+                            }
+                        ),
+                        json.dumps(
+                            {
+                                "app_id": "123",
+                                "app_name": "Second",
+                                "app_description": "Description.",
+                            }
+                        ),
+                    ]
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            missing_field = root / "missing_field.json"
+            missing_field.write_text(
+                json.dumps({"123": {"app_name": "Test VR"}}),
+                encoding="utf-8",
+            )
+
+            with self.assertRaises(ValueError):
+                check_metadata_cache(duplicate, ["123"])
+            with self.assertRaises(ValueError):
+                check_metadata_cache(missing_field, ["123"])
+
+    def test_verify_external_embedding_cache_against_manifest(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            cache = root / "embedding_dict.json"
+            cache.write_bytes(b"frozen-cache")
+            manifest = root / "cache_manifest.json"
+            manifest.write_text(
+                json.dumps(
+                    {
+                        "artifact": "embedding_dict.json",
+                        "size_bytes": cache.stat().st_size,
+                        "sha256": sha256_file(cache),
+                        "entry_count": 1,
+                        "embedding_dimension": 2,
+                    }
+                ),
+                encoding="utf-8",
+            )
+
+            report = check_embedding_cache(cache, manifest)
+
+        self.assertEqual(report["entry_count"], 1)
+        self.assertEqual(report["embedding_dimension"], 2)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_run_ape.py

@@ -0,0 +1,105 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from approach.run_ape import (
+    _resolve_ape_checkpoint,
+    _verify_ape_checkpoint,
+    build_parser,
+    run,
+)
+
+
+class RunApeTests(unittest.TestCase):
+    def test_custom_checkpoint_requires_explicit_trust(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            checkpoint = Path(tmpdir) / "custom.pth"
+            checkpoint.write_bytes(b"not-the-released-checkpoint")
+            with self.assertRaises(ValueError):
+                _verify_ape_checkpoint(checkpoint)
+            _verify_ape_checkpoint(checkpoint, trust_custom_checkpoint=True)
+
+    def test_checkpoint_resolver_accepts_repo_relative_path(self):
+        repo_relative = Path("approach/ovod/APE/ape_d_model_final.pth")
+        resolved = _resolve_ape_checkpoint(Path("approach/ovod/APE"), repo_relative)
+
+        self.assertEqual(resolved, (Path.cwd() / repo_relative).resolve())
+
+    def test_configurable_runner_selects_shard_and_writes_isolated_output(self):
+        calls = []
+
+        def inference(**kwargs):
+            calls.append(kwargs)
+            return [
+                {
+                    "category_name": kwargs["text_prompt"].split(":", 1)[0],
+                    "bbox": [1, 2, 3, 4],
+                    "score": 0.9,
+                }
+            ]
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            questions = root / "questions.jsonl"
+            candidates = root / "candidates.jsonl"
+            output = root / "predictions.json"
+            questions.write_text(
+                "\n".join(
+                    [
+                        json.dumps({"question_id": 0, "image": "123_4.jpg"}),
+                        json.dumps({"question_id": 1, "image": "456_7.jpg"}),
+                    ]
+                )
+                + "\n"
+            )
+            candidates.write_text(
+                "\n".join(
+                    [
+                        json.dumps(
+                            {
+                                "question_id": 0,
+                                "text": '{"objects": {"button": "round red"}}',
+                            }
+                        ),
+                        json.dumps(
+                            {
+                                "question_id": 1,
+                                "text": {"objects": {"lever": "long silver"}},
+                            }
+                        ),
+                    ]
+                )
+                + "\n"
+            )
+            args = build_parser().parse_args(
+                [
+                    "--questions",
+                    str(questions),
+                    "--candidates",
+                    str(candidates),
+                    "--images-dir",
+                    str(root / "images"),
+                    "--output",
+                    str(output),
+                    "--shard-index",
+                    "0",
+                    "--num-shards",
+                    "2",
+                ]
+            )
+
+            report = run(args, inference=inference)
+            shard_output = root / "predictions.shard00-of-02.json"
+
+            self.assertTrue(shard_output.exists())
+            self.assertTrue((root / "predictions.shard00-of-02.progress.json").exists())
+            self.assertEqual(json.loads(shard_output.read_text())[0]["image_id"], 123004)
+
+        self.assertEqual(report["records_selected"], 1)
+        self.assertEqual(len(calls), 1)
+        self.assertEqual(calls[0]["confidence_threshold"], 0.15)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_run_vlm.py

@@ -0,0 +1,83 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from approach.run_vlm import build_parser, run
+
+
+class RunVlmTests(unittest.TestCase):
+    def test_configurable_vlm_runner_writes_selected_records(self):
+        calls = []
+
+        def processor(profile, question, image_path, ablation, key_index):
+            calls.append((profile, question, Path(image_path).name, ablation, key_index))
+            return {"objects": {"button": "round red"}}
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            questions = root / "questions.jsonl"
+            questions.write_text(
+                "\n".join(
+                    [
+                        json.dumps({"question_id": 0, "image": "123_4.jpg", "text": "mine"}),
+                        json.dumps({"question_id": 1, "image": "456_7.jpg", "text": "mine"}),
+                    ]
+                )
+                + "\n"
+            )
+            args = build_parser().parse_args(
+                [
+                    "--questions",
+                    str(questions),
+                    "--images-dir",
+                    str(root / "images"),
+                    "--output",
+                    str(root / "answers.jsonl"),
+                    "--start-index",
+                    "0",
+                    "--end-index",
+                    "1",
+                    "--profile",
+                    "paper_claude",
+                ]
+            )
+
+            report = run(args, processor=processor)
+            output = root / "answers.rows0-1.jsonl"
+            answer = json.loads(output.read_text().strip())
+
+        self.assertEqual(report["completed"], 1)
+        self.assertEqual(answer["question_id"], 0)
+        self.assertEqual(answer["model_id"], "anthropic/claude-3.5-sonnet")
+        self.assertEqual(calls[0][:4], ("paper_claude", "mine", "123_4.jpg", False))
+
+    def test_default_processor_uses_app_metadata_cache_without_live_fallback(self):
+        from approach.vlm.gpt4v.gpt4v import get_steam_app_data, load_app_metadata_cache
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            root = Path(tmpdir)
+            cache_path = root / "metadata.json"
+            cache_path.write_text(
+                json.dumps(
+                    {
+                        "123": {
+                            "app_name": "Test VR",
+                            "app_description": "Synthetic cache-only description.",
+                        }
+                    }
+                ),
+                encoding="utf-8",
+            )
+            cache = load_app_metadata_cache(cache_path)
+
+        self.assertEqual(
+            get_steam_app_data("123", "123_4.jpg", cache),
+            ("Test VR", "Synthetic cache-only description."),
+        )
+        with self.assertRaises(KeyError):
+            get_steam_app_data("456", "456_7.jpg", cache)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_semantic_matching_cache.py

@@ -0,0 +1,121 @@
+import importlib.util
+import json
+import os
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import numpy  # noqa: F401 - preload C extension before dynamic module reloads.
+
+
+MODULE_PATH = (
+    Path(__file__).resolve().parents[1]
+    / "evaluation"
+    / "pycocotools_ovod"
+    / "semantic_matching.py"
+)
+
+
+class _FakeEmbeddings:
+    def __init__(self):
+        self.calls = []
+
+    def create(self, input, model):
+        self.calls.append((list(input), model))
+
+        class Response:
+            def model_dump(self_inner):
+                return {
+                    "data": [
+                        {"embedding": [1.0, 0.0]},
+                        {"embedding": [1.0, 0.0]},
+                    ]
+                }
+
+        return Response()
+
+
+def load_semantic_matching(cache_path, *, offline=False, readonly=False, api_key=None, fake_openai=None):
+    fake_openai = fake_openai or types.SimpleNamespace(embeddings=_FakeEmbeddings())
+
+    with mock.patch.dict(
+        os.environ,
+        {
+            "ORIENTER_EMBEDDING_CACHE": str(cache_path),
+            "ORIENTER_EMBEDDING_OFFLINE": "1" if offline else "",
+            "ORIENTER_EMBEDDING_READONLY": "1" if readonly else "",
+            "ZHIPU_API_KEY": api_key or "",
+        },
+        clear=False,
+    ), mock.patch.dict(sys.modules, {"openai": fake_openai}):
+        name = f"semantic_matching_under_test_{id(cache_path)}_{offline}_{readonly}"
+        spec = importlib.util.spec_from_file_location(name, MODULE_PATH)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        return module, fake_openai
+
+
+class SemanticMatchingCacheTests(unittest.TestCase):
+    def test_offline_cache_hit_does_not_call_api(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "embedding_dict.json"
+            cache_path.write_text(json.dumps({"Button": [1.0, 0.0]}))
+
+            module, fake_openai = load_semantic_matching(cache_path, offline=True)
+
+            self.assertTrue(module.is_semantic_match("button", "button", eval_dimension="s"))
+            self.assertEqual(fake_openai.embeddings.calls, [])
+
+    def test_offline_cache_miss_fails_before_api_call(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "embedding_dict.json"
+            cache_path.write_text(json.dumps({"Known": [1.0, 0.0]}))
+
+            module, fake_openai = load_semantic_matching(cache_path, offline=True)
+
+            with self.assertRaisesRegex(RuntimeError, "ORIENTER_EMBEDDING_OFFLINE=1"):
+                module.is_semantic_match("known", "missing", eval_dimension="s")
+
+            self.assertEqual(fake_openai.embeddings.calls, [])
+            self.assertFalse((Path(tmpdir) / "embedding_dict.json.save").exists())
+
+    def test_invalid_cache_is_rejected_at_load_time(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "embedding_dict.json"
+            cache_path.write_text(json.dumps({"A": [1.0, 0.0], "B": [1.0]}))
+
+            with self.assertRaisesRegex(ValueError, "dimension"):
+                load_semantic_matching(cache_path, offline=True)
+
+    def test_non_finite_cache_value_is_rejected_at_load_time(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "embedding_dict.json"
+            cache_path.write_text(json.dumps({"A": [1.0, float("nan")]}))
+
+            with self.assertRaisesRegex(ValueError, "non-finite"):
+                load_semantic_matching(cache_path, offline=True)
+
+    def test_readonly_cache_does_not_write_after_api_fill(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "embedding_dict.json"
+            cache_path.write_text(json.dumps({}))
+            original = cache_path.read_text()
+
+            module, fake_openai = load_semantic_matching(
+                cache_path,
+                readonly=True,
+                api_key="fake-key",
+            )
+            module.SAVE_INTERVAL = 1
+
+            self.assertTrue(module.is_semantic_match("alpha", "beta", eval_dimension="s"))
+            self.assertEqual(fake_openai.embeddings.calls, [(["Alpha", "Beta"], "embedding-3")])
+            self.assertEqual(cache_path.read_text(), original)
+            self.assertFalse((Path(tmpdir) / "embedding_dict.json.save").exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_vlm_adapter.py

@@ -0,0 +1,52 @@
+import tempfile
+import unittest
+import json
+from pathlib import Path
+from unittest import mock
+
+from approach.vlm.gpt4v import gpt4v
+
+
+class VlmAdapterTests(unittest.TestCase):
+    def test_app_metadata_cache_fails_on_missing_app_id(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            cache_path = Path(tmpdir) / "apps.json"
+            cache_path.write_text(json.dumps({"123": {"app_name": "App", "app_description": "Desc"}}))
+            cache = gpt4v.load_app_metadata_cache(cache_path)
+
+        self.assertEqual(gpt4v.get_steam_app_data("123", "123_4.jpg", cache), ("App", "Desc"))
+        with self.assertRaises(KeyError):
+            gpt4v.get_steam_app_data("456", "456_1.jpg", cache)
+
+    def test_process_image_uses_selected_openrouter_profile_and_real_mime_type(self):
+        client = mock.Mock()
+        client.complete_json.return_value = {"objects": {"button": "round red"}}
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            image_path = Path(tmpdir) / "123_4.png"
+            image_path.write_bytes(b"not-decoded-by-the-adapter")
+            with mock.patch.object(
+                gpt4v,
+                "get_steam_app_data",
+                return_value=("Test App", "Test description"),
+            ), mock.patch.object(
+                gpt4v,
+                "OpenAICompatibleChatClient",
+                return_value=client,
+            ) as client_class:
+                result = gpt4v.process_image(
+                    "default",
+                    "describe",
+                    str(image_path),
+                    False,
+                    0,
+                )
+
+        self.assertEqual(result["objects"]["button"], "round red")
+        self.assertEqual(client_class.call_args.args[0].provider, "openrouter")
+        encoded_image = client.complete_json.call_args.args[1][0]
+        self.assertEqual(encoded_image.media_type, "image/png")
+
+
+if __name__ == "__main__":
+    unittest.main()

util/llm_gemini.py

@@ -0,0 +1,14 @@
+import os
+import google.generativeai as genai
+
+API_KEY = os.environ.get("GOOGLE_API_KEY", "[GOOGLE_API_KEY]")
+
+genai.configure(api_key=API_KEY)
+model = genai.GenerativeModel('gemini-pro')
+
+def generate_content(prompt, _place_holder = None):
+    response = model.generate_content(prompt)
+    return response.text
+
+if __name__ == "__main__":
+    print(generate_content("What are you?"))

util/llm_third_party.py

@@ -0,0 +1,60 @@
+import os
+import requests
+import json
+import llm_gemini
+
+AVAILABLE_MODELS = [
+    "claude-3-haiku",
+    "claude-3-sonnet",
+    "claude-3-opus",
+    "gemini-pro",
+    "gpt-3.5-turbo",
+    "gpt-4-turbo",
+    "gpt-4",
+]
+
+MODEL = "claude-3-haiku"
+
+URL = "https://api.theb.ai/v1/chat/completions"
+# URL = "https://api.baizhi.ai/v1/chat/completions"
+API_KEY = os.environ.get("THEB_API_KEY", "[THEB_API_KEY]")
+
+
+def generate_content(prompt, model=MODEL, api_key=API_KEY):
+    assert model in AVAILABLE_MODELS, f"Model {model} not available"
+    if model == "gemini-pro":
+        return llm_gemini.generate_content(prompt)
+    payload = json.dumps({
+        "model": model,
+        "messages": [
+            {
+            "role": "user",
+            "content": prompt
+            }
+        ],
+        "stream": False
+        })
+    headers = {
+        'Authorization': f'Bearer {api_key}',
+        'Content-Type': 'application/json'
+    }
+
+    response = requests.request("POST", URL, headers=headers, data=payload, timeout=10)
+    if response.status_code != 200:
+        retry_count = 0
+        while retry_count < 5:
+            response = requests.request("POST", URL, headers=headers, data=payload, timeout=10)
+            if response.status_code == 200:
+                break
+            else:
+                retry_count += 1
+        if retry_count == 5:
+            raise Exception(f"Failed to get completion from LLM. Status code: {response.status_code}. Response: {response.text}")
+
+    # print(response)
+
+    return response.json()['choices'][0]['message']['content']
+
+
+if __name__ == "__main__":
+    print(generate_content("What are you?", "claude-3-haiku"))