Add files using upload-large-folder tool

approach/ape_stage.py

@@ -0,0 +1,150 @@
+import json
+import string
+from pathlib import Path
+from typing import Callable, Dict, Iterable, List, Mapping, Optional, Tuple
+
+from approach.pipeline_utils import (
+    enrich_ape_results,
+    parse_orienter_image_name,
+    resolve_image_path,
+    write_json_atomic,
+)
+
+
+InferenceFn = Callable[..., List[dict]]
+
+
+def extract_image_id(image_name: str) -> int:
+    return parse_orienter_image_name(Path(image_name).name)[2]
+
+
+def build_referring_expressions(objects: Mapping[str, object]) -> List[str]:
+    translator = str.maketrans("", "", string.punctuation)
+    expressions = []
+    for name, description in objects.items():
+        if isinstance(description, str):
+            text = description
+        elif isinstance(description, Mapping):
+            text = " ".join(str(value) for value in description.values())
+        else:
+            text = str(description)
+        expressions.append(f"{name}: {text.translate(translator)}")
+    return expressions
+
+
+def _load_existing(output_path: Path) -> List[dict]:
+    if not output_path.exists():
+        return []
+    with output_path.open() as file:
+        data = json.load(file)
+    if not isinstance(data, list):
+        raise ValueError(f"Existing prediction file must contain a list: {output_path}")
+    return data
+
+
+def _completed_image_ids(results: Iterable[dict]) -> set:
+    return {item["image_id"] for item in results if "image_id" in item}
+
+
+def _progress_path(output_path: Path) -> Path:
+    return output_path.with_name(f"{output_path.stem}.progress.json")
+
+
+def _load_progress(output_path: Path) -> set:
+    path = _progress_path(output_path)
+    if not path.exists():
+        return set()
+    with path.open() as file:
+        data = json.load(file)
+    return set(data.get("completed_image_ids", []))
+
+
+def _get_question(questions: Mapping[object, Mapping[str, object]], question_id):
+    if question_id in questions:
+        return questions[question_id]
+    question_id_str = str(question_id)
+    if question_id_str in questions:
+        return questions[question_id_str]
+    try:
+        question_id_int = int(question_id)
+    except (TypeError, ValueError):
+        question_id_int = None
+    if question_id_int in questions:
+        return questions[question_id_int]
+    raise KeyError(question_id)
+
+
+def run_ape_stage(
+    records: Iterable[dict],
+    questions: Mapping[object, Mapping[str, object]],
+    images_dir,
+    output_path,
+    inference: InferenceFn,
+    inference_kwargs: Optional[Dict[str, object]] = None,
+    error_path=None,
+    resume: bool = True,
+) -> Tuple[List[dict], List[dict]]:
+    output_path = Path(output_path)
+    error_path = Path(error_path) if error_path is not None else None
+    images_dir = Path(images_dir)
+    inference_kwargs = dict(inference_kwargs or {})
+    results = _load_existing(output_path) if resume else []
+    completed = _completed_image_ids(results)
+    if resume:
+        completed.update(_load_progress(output_path))
+    errors = []
+    write_json_atomic(str(output_path), results)
+    write_json_atomic(
+        str(_progress_path(output_path)),
+        {"completed_image_ids": sorted(completed)},
+    )
+
+    for record in records:
+        question_id = record["question_id"]
+        question = _get_question(questions, question_id)
+        image_name = question["image"]
+        image_id = extract_image_id(image_name)
+        if image_id in completed:
+            continue
+
+        text = record.get("text")
+        if not text:
+            continue
+        objects = text["objects"]
+        expressions = build_referring_expressions(objects)
+        image_path = resolve_image_path(images_dir, image_name)
+
+        try:
+            ape_results = inference(
+                input_path=str(image_path),
+                text_prompt=", ".join(expressions),
+                **inference_kwargs,
+            )
+            enriched = enrich_ape_results(ape_results, image_name, extract_image_id)
+            results.extend(enriched)
+            completed.add(image_id)
+            write_json_atomic(str(output_path), results)
+            write_json_atomic(
+                str(_progress_path(output_path)),
+                {"completed_image_ids": sorted(completed)},
+            )
+        except Exception as exc:
+            errors.append(
+                {
+                    "question_id": question_id,
+                    "image": image_name,
+                    "error_type": type(exc).__name__,
+                    "message": str(exc),
+                }
+            )
+            if error_path is not None:
+                write_json_atomic(str(error_path), errors)
+
+    write_json_atomic(str(output_path), results)
+    write_json_atomic(
+        str(_progress_path(output_path)),
+        {"completed_image_ids": sorted(completed)},
+    )
+    if error_path is not None:
+        write_json_atomic(str(error_path), errors)
+    return results, errors

approach/llm/gpt.py

@@ -0,0 +1,90 @@
+import os
+import json
+import time
+import requests
+from bs4 import BeautifulSoup
+from approach.config import get_model_profile
+from approach.pipeline_utils import write_json_atomic
+from approach.providers import OpenAICompatibleChatClient
+
+DEFAULT_PROFILE = os.environ.get("ORIENT_MODEL_PROFILE", "default")
+
+
+def get_steam_app_data(app_id, image_name):
+    url = f"https://store.steampowered.com/app/{app_id}"
+
+    try:
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()  # Raise an HTTPError if the HTTP request returned an unsuccessful status code
+
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        app_name = soup.find("div", {"id": "appHubAppName"}).get_text(strip=True)
+        short_desc = soup.find("div", class_="game_description_snippet").get_text(strip=True)
+        long_desc = soup.find("div", {"id": "game_area_description"}).get_text(strip=True)
+
+        return app_name, short_desc + " " + long_desc
+    except Exception as e:
+        print(f"Error fetching app data for app_id {app_id} of image {image_name}. Error: {e}")
+        return "", ""
+
+def generate_prompt(app_name, app_description, app_current_view):
+    return (f"You are a virtual reality game player. Currently, you are playing a game named {app_name} "
+            f"with the following description: {app_description}. Now in your field of view, you can see that: {app_current_view}. "
+            "Please infer what virtual objects you can see currently based on the current view and the app description.")
+
+def infer_object_candidates(
+    vlm_question_path,
+    vlm_answer_path,
+    llm_candidate_path,
+    model_profile=DEFAULT_PROFILE,
+):
+    profile = get_model_profile(model_profile)
+    client = OpenAICompatibleChatClient(profile)
+    # Read and process the two jsonl files
+    with open(vlm_answer_path, "r") as f_a:
+        file_a_data = [json.loads(line) for line in f_a.readlines()]
+
+    with open(vlm_question_path, "r") as f_b:
+        file_b_data = {json.loads(line)["question_id"]: json.loads(line) for line in f_b.readlines()}
+        # file_b_dict = {json.loads(line)["question_id"]: json.loads(line)["image"].split("_")[0] for line in f_b.readlines()}
+
+    results = {}
+
+    index = 0
+    for line_data_a in file_a_data:
+        print(index)
+        index += 1
+
+        question_id = line_data_a["question_id"]
+        app_current_view = line_data_a["text"]
+
+        line_data_b = file_b_data.get(question_id, {})
+        app_id = line_data_b.get("image", "").split("_")[0]
+        # app_id = file_b_dict.get(question_id, "")
+        # line_data_b = file_b_dict.get(question_id, {})
+
+        if app_id:
+            app_name, app_description = get_steam_app_data(app_id, line_data_b.get("image", ""))
+            message_content = generate_prompt(app_name, app_description, app_current_view)
+            try:
+                payload = client.build_payload(message_content)
+                completion = client.chat_completion(payload)
+                content = completion["choices"][0]["message"]["content"].strip()
+                inferred_objects = [item.strip() for item in content.split(";") if item.strip()]
+
+                results[question_id] = {
+                    "vlm_question": line_data_a,
+                    "vlm_answer": line_data_b,
+                    "app_name": app_name,
+                    "app_description": app_description,
+                    "app_current_view": app_current_view,
+                    "inferred_object_candidates": inferred_objects
+                }
+            except Exception as e:
+                print(f"Error generating completion for question_id {question_id}. Error: {e}")
+
+        time.sleep(20)
+
+    # Save the results to a json file
+    write_json_atomic(llm_candidate_path, results)

approach/llm/gpt_polling.py

@@ -0,0 +1,268 @@
+import os
+import json
+import time
+import requests
+from bs4 import BeautifulSoup
+from approach.config import get_model_profile
+from approach.providers import OpenAICompatibleChatClient
+
+DEFAULT_PROFILE = os.environ.get("ORIENT_MODEL_PROFILE", "default")
+
+
+def get_steam_app_data(app_id, image_name):
+    url = f"https://store.steampowered.com/app/{app_id}"
+
+    try:
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        app_name = soup.find("div", {"id": "appHubAppName"}).get_text(strip=True)
+        short_desc = soup.find("div", class_="game_description_snippet").get_text(strip=True)
+        long_desc = soup.find("div", {"id": "game_area_description"}).get_text(strip=True)
+
+        return app_name, short_desc + " " + long_desc
+    except Exception as e:
+        print(f"Error fetching app data for app_id {app_id} of image {image_name}. Error: {e}")
+        return "", ""
+
+
+def save_results_to_file(llm_candidate_path, results):
+    """Utility function to save results to a file."""
+    with open(llm_candidate_path, "w") as out_file:
+        json.dump(results, out_file, indent=4)
+
+
+def generate_prompt(app_name, app_description, app_current_view, ablation):
+    # return (f"You are a virtual reality game player. Currently, you are playing a game named {app_name} "
+    #         f"with the following description: . Now in your field of view, you can see that: {app_current_view}. "
+    #         "Please infer what virtual objects you can see currently based on the current view and the app description. Let's think step by step.\n"
+    #         "List the virtual objects one by one, seperated by semicolons"
+    #         f"App description: {app_description}")
+
+    # return f'''
+    # ** Context and Task **
+    # You are a virtual reality game player. Currently, you are playing a game named {app_name} with the following description. Now in your field of view from the VR headset, you can see that: {app_current_view}. Please infer what virtual objects you can see currently based on the current view and the app description.\n
+    # Output the inferred objects one by one, seperated by semicolons, strictly following the format: object1;object2. E.g., "door;desk"\n
+    # NO DOT OUTPUT any other content.\n
+
+    # ** Chain-of-Thought Reasoning Demonstration **
+    # (1) For a tree object in a tree-planting game, the users need to pick up and plant the tree, thus the object is interactable.
+    # (2) But for a tree object appeard in the background scenery of a fishing-only game, the object is highly likely non-interactable.\n
+
+    # ** Data **
+    # The description of the app:
+    # <app_description>.
+    # '''
+
+    # return f'''
+    # ** Context and Task **
+    # You are a virtual reality game player. Currently, you are playing a game named {app_name} with the following description. Now in your field of view from the VR headset, you can see that: {app_current_view}. Please extract information and infer what virtual objects you can see currently based on the current view and the app description.\n
+    # Output the inferred objects one by one, seperated by semicolons, strictly following the format: object1;object2. E.g., "door;desk"\n
+    # NO DOT OUTPUT any other content.\n
+
+    # ** Data **
+    # The description of the app:
+    # <app_description>.
+    # '''
+
+    if ablation:
+        return f'''
+        Please extract objects from the following description: {app_current_view}
+        Output the extracted objects in JSON format: {{"objects": ["object1", "object2"]}}. E.g., {{"objects": ["door", "desk"]}}\n
+        NO DOT OUTPUT any other content besides JSON.\n
+        '''
+    elif not ablation:
+        return f'''
+        --- Context and Task ---
+        You are a virtual reality game player. Currently, you are playing a game named {app_name} with the following description. Now in your field of view from the VR headset, you can see the current VR scenario: {app_current_view}. Please extract information and infer what virtual objects you can see currently based on the current view and the app description.\n
+
+        --- Instructions and Demonstrations ---
+        (1) If there are multiple similar objects, please output all objects one by one. E.g., if there are several pens on the table, please output several "pen" instead of a single "pens".
+        (2) Provide detailed object description with texts on them (if any). For example, for a button or option with text on it (e.g., begin), you need to output "begin button" or "begin option".
+        (3) Ignore the virtual objects of user-controlled VR devices, e.g., controllers, hands and gloves, which are near the users in screenshot.
+        (4) Format: Output the inferred objects in JSON format: {{"objects": ["object1", "object2"]}}. E.g., {{"objects": ["door", "desk"]}}\n
+        NO DOT OUTPUT any other content besides JSON.\n
+
+        --- Data ---
+        The description of the app:
+        {app_description}.
+        '''
+
+def generate_i_prompt(inferred_objects, app_current_view, ablation):
+    if ablation:
+        return f'''
+        For the following objects: {inferred_objects}. Please identify those user-interactable objects in VR games.
+        Output the extracted objects in JSON format: {{"objects": ["object1", "object2"]}}. E.g., {{"objects": ["door", "desk"]}}\n
+        NO DOT OUTPUT any other content besides JSON.\n
+        '''
+    elif not ablation:
+        return f'''
+        --- Context and Task ---
+        You are a virtual reality game player. From your previous response, you find that the following objects exist in the screenshot: {inferred_objects}.
+        Please perform reasoning to identify the user-interactable objects, with which users can use VR devices like handheld controllers to interact, in the current VR scene screenshot. Please perform reasoning based on the app description and current VR scenario.\n
+
+        --- Instructions and Demonstrations ---
+        (1) For a tree object in a tree-planting game, the users need to pick up and plant the tree, thus the object is interactable.
+        (2) But for a tree object appeard in the background scenery of a fishing-only game, the object is highly likely non-interactable.\n
+        Output the interactable objects in JSON format: {{"objects": ["object1", "object2"]}}. E.g., {{"objects": ["door", "desk"]}}\n
+        NO DOT OUTPUT any other content besides JSON.\n
+
+        --- Data ---
+        The current VR scenario in the screenshot:
+        {app_current_view}
+        '''
+
+
+# def infer_object_candidates(vlm_question_path, vlm_answer_path, llm_candidate_path):
+def infer_objects(vlm_question_path, vlm_answer_path, llm_candidate_path, ablation, model_profile=DEFAULT_PROFILE):
+    profile = get_model_profile(model_profile)
+    client = OpenAICompatibleChatClient(profile)
+
+    with open(vlm_answer_path, "r") as f_a:
+        file_a_data = [json.loads(line) for line in f_a.readlines()]
+
+    with open(vlm_question_path, "r") as f_b:
+        file_b_data = {json.loads(line)["question_id"]: json.loads(line) for line in f_b.readlines()}
+
+    results = {}
+    index = 0
+
+    for line_data_a in file_a_data:
+        print(index)
+        index += 1
+
+        # TODO: Proxy pool
+        # proxy = get_proxy()
+        # if not proxy:
+        #     print("Failed to fetch proxy.")
+        #     continue
+
+        question_id = line_data_a["question_id"]
+        app_current_view = line_data_a["text"]
+        line_data_b = file_b_data.get(question_id, {})
+        app_id = line_data_b.get("image", "").split("_")[0]
+
+        if app_id:
+            app_name, app_description = get_steam_app_data(app_id, line_data_b.get("image", ""))
+            message_content = generate_prompt(app_name, app_description, app_current_view, ablation)
+            # switch_api_key()  # Use the next API key in the list
+            try:
+                # TODO: Proxy pool (f"http://{proxy}")
+                completion = client.complete_json(message_content)
+                inferred_objects = completion["objects"]
+                inferred_objects = [io.strip() for io in inferred_objects]
+
+                # wrong_flag = False
+                # for io in inferred_objects:
+                #     if io.startswith('Based on') or len(io) > 70:
+                #         print('Based on')
+                #         wrong_flag = True
+                #         break
+                # if wrong_flag:
+                #     continue
+                print(inferred_objects)
+
+                i_completion = client.complete_json_messages([
+                    {'role': 'user', 'content': message_content},
+                    {'role': 'assistant', 'content': json.dumps(completion)},
+                    {'role': 'user', 'content': generate_i_prompt(inferred_objects, app_current_view, ablation)}
+                ])
+                interactable_objects = i_completion["objects"]
+                interactable_objects = [io.strip() for io in interactable_objects]
+                # TODO: !!
+                if len(interactable_objects) == 0:
+                    interactable_objects = ['\n']
+                print(interactable_objects)
+
+                results[question_id] = {
+                    "vlm_question": line_data_a,
+                    "vlm_answer": line_data_b,
+                    "app_name": app_name,
+                    "app_description": app_description,
+                    "app_current_view": app_current_view,
+                    "inferred_object_candidates": inferred_objects,
+                    "interactable_objects": interactable_objects
+                }
+
+                # Save the updated results to file after each successful API call
+                save_results_to_file(llm_candidate_path, results)
+
+            except Exception as e:
+                print(f"Error generating completion for question_id {question_id}. Error: {e}")
+
+        time.sleep(3)  # Sleep for 2 seconds before switching to the next API key
+
+
+# def infer_interactable_object(vlm_question_path, vlm_answer_path, llm_candidate_path):
+#     with open(vlm_answer_path, "r") as f_a:
+#         file_a_data = [json.loads(line) for line in f_a.readlines()]
+
+#     with open(vlm_question_path, "r") as f_b:
+#         file_b_data = {json.loads(line)["question_id"]: json.loads(line) for line in f_b.readlines()}
+
+#     results = {}
+#     index = 0
+
+#     for line_data_a in file_a_data:
+#         print(index)
+#         index += 1
+
+#         # TODO: Proxy pool
+#         # proxy = get_proxy()
+#         # if not proxy:
+#         #     print("Failed to fetch proxy.")
+#         #     continue
+
+#         question_id = line_data_a["question_id"]
+#         app_current_view = line_data_a["text"]
+#         line_data_b = file_b_data.get(question_id, {})
+#         app_id = line_data_b.get("image", "").split("_")[0]
+
+#         if app_id:
+#             app_name, app_description = get_steam_app_data(app_id, line_data_b.get("image", ""))
+#             message_content = generate_prompt(app_name, app_description, app_current_view)
+#             # switch_api_key()  # Use the next API key in the list
+#             try:
+#                 # TODO: Proxy pool (f"http://{proxy}")
+#                 completion = openai.ChatCompletion.create(
+#                     model='gemini-3-flash-preview-nothinking',
+#                     temperature=0,
+#                     messages=[{'role': 'user', 'content': message_content}]
+#                 )
+#                 print(completion)
+#                 # inferred_objects = completion['choices'][0]['message']['content'].strip().split(";")
+#                 inferred_objects = json.loads(completion['choices'][0]['message']['content'].strip())['objects']
+#                 inferred_objects = [io.strip() for io in inferred_objects]
+
+#                 # wrong_flag = False
+#                 # for io in inferred_objects:
+#                 #     if io.startswith('Based on') or len(io) > 70:
+#                 #         print('Based on')
+#                 #         wrong_flag = True
+#                 #         break
+#                 # if wrong_flag:
+#                 #     continue
+#                 print(inferred_objects)
+
+#                 results[question_id] = {
+#                     "vlm_question": line_data_a,
+#                     "vlm_answer": line_data_b,
+#                     "app_name": app_name,
+#                     "app_description": app_description,
+#                     "app_current_view": app_current_view,
+#                     "inferred_object_candidates": inferred_objects
+#                 }
+
+#                 # Save the updated results to file after each successful API call
+#                 save_results_to_file(llm_candidate_path, results)
+
+#             except Exception as e:
+#                 print(f"Error generating completion for question_id {question_id}. Error: {e}")
+
+#         time.sleep(0.5)  # Sleep for 2 seconds before switching to the next API key
+
+
+if __name__ == "__main__":
+    infer_object_candidates("path_to_vlm_question", "path_to_vlm_answer", "path_to_llm_candidate")

approach/method.py

@@ -0,0 +1,732 @@
+import os
+import sys
+import json
+import time
+import random
+import string
+import argparse
+import copy
+import logging
+from tqdm import tqdm
+import PIL
+from PIL import ImageFile
+PIL.ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(BASE_DIR)
+from approach.pipeline_utils import (
+    enrich_ape_results,
+    output_path_for_selection,
+    run_optional_reflection,
+    select_jsonl_lines,
+    write_json_atomic,
+)
+
+os.environ['TORCH_HOME'] = './.cache'
+os.environ['HF_HOME'] = './.cache'
+
+
+vlm = 'llava7b'
+llm = 'gpt_3.5_turbo'
+ovod = 'grounding_dino'
+
+vlm_prompt = ''
+
+### Configurable
+host_device = os.getenv('ORIENTER_LEGACY_HOST', 'local')
+perspective = 'all_perspective'
+# perspective = 'direct_back'
+# perspective = 'direct_front'
+# perspective = 'direct_side'
+# perspective = 'direct_top'
+# perspective = 'eyelevel'
+# perspective = 'overlook'
+### Configurable
+
+llava_path = os.path.join(BASE_DIR, 'approach/vlm/LLaVA/llava/eval')
+grounding_dino_path = os.path.join(BASE_DIR, 'approach/ovod/GroundingDINO')
+ape_path = os.path.join(BASE_DIR, 'approach/ovod/APE')
+# TODO: data on the CUHK server
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_merged/images/interactable')
+# icse
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_merged/images/test_set_merged')
+# icse_rebuttal
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_angleview/images/all_perspective')
+# fse
+images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_det/images/union3')
+
+
+# def generate_grounding_dino_command(ovod_image_path, ovod_output_dir, ovod_candidates):
+#     gdino_object_str = ' . '.join(ovod_candidates)
+
+#     gdino_command = f'''
+#     python demo/inference_on_a_image.py \
+#         -c groundingdino/config/GroundingDINO_SwinT_OGC.py \
+#         -p weights/groundingdino_swint_ogc.pth \
+#         -i {ovod_image_path} \
+#         -o "{ovod_output_dir}" \
+#         -t "{gdino_object_str}"
+#     '''
+#     return gdino_command
+
+
+def generate_question_file(img_folder, dst_path):
+    # Ensure the directory exists
+    if not os.path.exists(img_folder):
+        print(f"Error: Directory {img_folder} does not exist.")
+        return
+
+    # List all files in the directory
+    all_files = os.listdir(img_folder)
+
+    # Filter out files that are not images (based on extension). You can add more if needed.
+    image_extensions = [".jpg", ".jpeg", ".png", ".bmp", ".gif", ".tiff"]
+    image_files = [f for f in all_files if any(f.lower().endswith(ext) for ext in image_extensions)]
+
+    # Open the output file for writing
+    with open(dst_path, 'w') as out_file:
+        for index, img_file in enumerate(image_files):
+            data = {
+                "question_id": index,
+                "image": img_file,
+                "text": vlm_prompt,
+                "category": "detail"
+            }
+            out_file.write(json.dumps(data) + '\n')
+
+    print(f"Processed {len(image_files)} images. Output saved to {dst_path}.")
+
+
+def generate_answer_id(length=20):
+    # Define the characters that can be used in the string
+    characters = string.ascii_letters + string.digits
+    # Generate a random string of the specified length
+    answer_id = ''.join(random.choice(characters) for _ in range(length))
+    return answer_id
+
+
+def method(
+    vlm=vlm,
+    llm=llm,
+    ovod=ovod,
+    start_index=None,
+    end_index=None,
+    shard_index=None,
+    num_shards=None,
+    enable_reflection=False,
+    reflection_profile="default",
+    max_reflection_iterations=10,
+    questions_path=None,
+    candidates_path=None,
+    images_path=None,
+    output_path=None,
+    ape_root=None,
+    ape_checkpoint=None,
+):
+    images_dir = images_path or globals()["images_dir"]
+    active_ape_path = ape_root or globals()["ape_path"]
+    active_ape_checkpoint = ape_checkpoint or "./ape_d_model_final.pth"
+    # vlm_question_path = os.path.join(BASE_DIR, 'approach/vlm/sampled_vlm_questions.jsonl')
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_vlm_questions_from671.jsonl')
+    # icse
+    # complete_vlm_question_abl_i_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_complete_vlm_questions_ablation_interactability.jsonl')
+
+    # icse
+    # complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_complete_vlm_questions.jsonl')
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/gpt4v_failure_questions.jsonl')
+    # icse_rebuttal
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_vlm_questions.jsonl')
+    # complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_complete_vlm_questions.jsonl')
+    # fse
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/fse_union3_{vlm}_questions.jsonl')
+    complete_vlm_question_path = questions_path or os.path.join(BASE_DIR, f'approach/vlm/realfse_union3_complete_{vlm}_questions.jsonl')
+    vlm_question_path = complete_vlm_question_path
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/gpt4v_failure_questions.jsonl')
+    # playground/data/coco2014_val_qa_eval/qa90_questions.jsonl
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_slowerspeed_1.jsonl')
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_higherspeed_r3_1_from195.jsonl')
+
+    # icse
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/cuhk_icse_test_set_merged_{vlm}_answer.jsonl')
+    # llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer.jsonl')
+    # gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_ablation_no_interactability_r1_0.jsonl')
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_cuhk_icse_test_set_merged_gpt4v_answer_higherspeed_r3_0.jsonl')
+    # interactability_abl_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_hit_icse_test_set_merged_gemini_answer_ablation_no_interactability_r1_0.jsonl')
+    # icse_rebuttal
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # fse
+    vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_uninon3_{vlm}_answer.jsonl')
+    llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_fse_uninon3_{vlm}_answer.jsonl')
+    gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_union3_{vlm}_answer.jsonl')
+
+    # icse
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_icse_rbt_{perspective}_gpt4v_answer.jsonl')
+    # icse_rebuttal
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_{vlm}_answer_265_528.jsonl')
+    # fse
+    gpt4_results_answer_path = candidates_path or os.path.join(BASE_DIR, f'approach/vlm/realfse_uninon3_gpt4v_answer.jsonl')
+    # print(gpt4_results_answer_path)
+    # interactability_abl_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_icse_rbt_{perspective}_answer.jsonl')
+
+    # icse
+    # # /path/to/answer-file-our.jsonl
+    # # llm_candidate_path = os.path.join(BASE_DIR, f'approach/llm/{vlm}_{llm}_cancidate_objects.json')
+    # interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/icse_test_set_merged_{vlm}_{llm}_interactable_objects.json')
+    # ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/icse_test_set_merged_{ovod}')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_object_bbox_gpu3_2.json')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_interactability_object_bbox_gpu0.json')
+    # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_feedback_object_bbox_gpu2.json')
+
+    # icse_rebuttal
+    # interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/icse_rbt_{perspective}_{vlm}_{llm}_interactable_objects.json')
+    # ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/icse_rbt_{perspective}_{ovod}')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_object_bbox_gpu3_2.json')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_interactability_object_bbox_gpu0.json')
+    # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_rbt_{perspective}_{vlm}_{llm}_{ovod}_object_bbox_gpu3_265_528.json')
+
+    gpu = '1-3'
+    # fse
+    interactable_object_path = candidates_path or os.path.join(BASE_DIR, f'approach/llm/realfse_union3_{vlm}_{llm}_interactable_objects.json') # TODO: Tentatively unused
+    ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/realfse_union3_{ovod}')
+    # GPU 0
+    base_output_path = output_path or os.path.join(BASE_DIR, f'approach/ovod/realfse_union3_{vlm}_{ovod}_object_bbox_gpu{gpu}.json')
+    oovd_object_bbox_path = output_path_for_selection(
+        base_output_path,
+        start_index=start_index,
+        end_index=end_index,
+        shard_index=shard_index,
+        num_shards=num_shards,
+    )
+
+    llava7b_model_path = os.path.join(BASE_DIR, 'approach/vlm/LLaVA/checkpoints/llava-v1.5-7b')
+    llava7b_cmd = f'''
+    python model_vqa.py \
+        --model-path {llava7b_model_path} \
+        --question-file \
+        {complete_vlm_question_path} \
+        --image-folder \
+        {images_dir} \
+        --answers-file \
+        {llava_vlm_answer_path}
+    '''
+
+    # STEP #0
+    # First time of running
+    # TODO: Check whether it is first-time running
+    # generate_question_file(images_dir, vlm_question_path)
+
+    # STEP #1
+    # VLM - Get local context
+    # print(f'VLM {vlm} analysis begins ...')
+    # vlm_start_time = time.time()
+    # logging.info(f'STEP #1 VLM {vlm} analysis started at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    # if vlm == 'llava7b':
+    #     original_path = os.path.dirname(__file__)
+    #     os.chdir(llava_path)
+    #     os.system(llava7b_cmd)
+    #     os.chdir(original_path)
+    # elif vlm == 'generate_q_file':
+    #     from approach.vlm.gpt4v.gpt4v import process_image_q
+    #     with open(vlm_question_path, 'r') as q_file, open(complete_vlm_question_path, 'w') as cq_file:  # Open answer file in append mode
+    #         qfile_lines = q_file.readlines()
+    #         key_idx = 0
+    #         for line in tqdm(qfile_lines):
+    #             key_idx = (key_idx + 1) % 4
+
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             if vlm == 'gpt4v_abl':
+    #                 gpt4v_ablation = True
+    #             elif vlm == 'gpt4v':
+    #                 gpt4v_ablation = False
+    #             gpt4v_q = process_image_q(image_question, image_path, key_idx) 
+    #             data = {
+    #                 "question_id": line_data['question_id'],
+    #                 "image": line_data['image'],
+    #                 "text": gpt4v_q,
+    #                 "category": "detail"
+    #             }
+    #             cq_file.write(json.dumps(data) + '\n')       
+    # elif vlm == 'gpt4v' or vlm == 'gpt4v_abl' or vlm == 'claude35sonnet' or vlm == 'gemini15pro':
+    #     from approach.vlm.gpt4v.gpt4v import process_image
+    #     with open(vlm_question_path, 'r') as q_file, open(vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+    #         qfile_lines = q_file.readlines()
+    #         key_idx = 0
+    #         for line in tqdm(qfile_lines):
+    #             key_idx = (key_idx + 1) % 4
+
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             gpt4v_ablation = False
+    #             if vlm == 'gpt4v_abl':
+    #                 gpt4v_ablation = True
+    #             elif vlm == 'gpt4v':
+    #                 gpt4v_ablation = False
+    #             gpt4v_res = process_image(vlm, image_question, image_path, gpt4v_ablation, key_idx)
+    #             print(gpt4v_res)
+    #             # time.sleep(3)
+
+    #             ans_item = {
+    #                 "question_id": line_data['question_id'], 
+    #                 "prompt": '', 
+    #                 "text": gpt4v_res, 
+    #                 "answer_id": generate_answer_id(), 
+    #                 "model_id": vlm, 
+    #                 "metadata": {}
+    #             }
+
+    #             a_file.write(json.dumps(ans_item) + '\n')  
+    #             a_file.flush()
+    # elif vlm == 'bing':
+    #     from approach.vlm.bing.bing import context_conversation
+    #     with open(vlm_question_path, 'r') as q_file, open(vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+    #         for line in tqdm(q_file):
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             bing_res = context_conversation(image_question, image_path)
+    #             print(bing_res)
+    #             time.sleep(15)
+
+    #             ans_item = {
+    #                 "question_id": line_data['question_id'], 
+    #                 "prompt": image_question, 
+    #                 "text": bing_res, 
+    #                 "answer_id": generate_answer_id(), 
+    #                 "model_id": vlm, 
+    #                 "metadata": {}
+    #             }
+
+    #             a_file.write(json.dumps(ans_item) + '\n')  
+    # elif vlm == 'gemini':
+    #     import pathlib
+    #     import textwrap
+
+    #     import google.generativeai as genai
+
+    #     GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "[GOOGLE_API_KEY]")
+    #     genai.configure(api_key=GOOGLE_API_KEY)
+
+    #     safety_settings = [
+    #         {
+    #             "category": "HARM_CATEGORY_DANGEROUS",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_HARASSMENT",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_HATE_SPEECH",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #     ]
+
+    #     # before fse
+    #     # model = genai.GenerativeModel('gemini-pro-vision')
+    #     # fse
+    #     model = genai.GenerativeModel('gemini-1.5-pro')
+
+    #     with open(vlm_question_path, 'r') as q_file, open(gemini_vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+    #         qfile_lines = q_file.readlines()[0:2]
+    #         for line in tqdm(qfile_lines):
+
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             gemini_response = model.generate_content([image_question, PIL.Image.open(image_path)], safety_settings=safety_settings)
+    #             gemini_response.resolve()
+    #             try:
+    #                 gemini_response = gemini_response.text
+    #                 ans_item = {
+    #                     "question_id": line_data['question_id'], 
+    #                     "prompt": image_question, 
+    #                     "text": gemini_response, 
+    #                     "answer_id": generate_answer_id(), 
+    #                     "model_id": vlm, 
+    #                     "metadata": {}
+    #                 }
+
+    #                 a_file.write(json.dumps(ans_item) + '\n')  
+    #                 a_file.flush()
+    #             except Exception as e:
+    #                 print(f"Error for image {image_path}. Error: {e}")                
+    #             # print(gemini_response)
+    #             time.sleep(30)
+
+
+
+    # elif vlm.endswith('_pass'):
+    #     print(f'Passing VLM {vlm} ...')
+    # else:
+    #     raise Exception('Unrecognized VLM!')
+
+    # logging.info(f'STEP #1 VLM {vlm} analysis completed at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    # logging.info(f'STEP #1 VLM {vlm} analysis total time taken: {time.time() - vlm_start_time} seconds')
+
+
+
+
+
+    # Get label candidates
+    # if llm == 'gpt_3.5_turbo':
+    #     from approach.llm.gpt_polling import infer_object_candidates
+    # elif llm == 'llama2':
+    #     from approach.llm.llama import infer_object_candidates
+
+
+    # STEP #3
+    # if llm == 'gpt_3.5_turbo' or llm == 'gpt_3.5_turbo_abl':
+    #     from approach.llm.gpt_polling import infer_objects
+
+    #     if llm == 'gpt_3.5_turbo_abl':
+    #         gpt35_ablation = True
+    #     elif llm == 'gpt_3.5_turbo':
+    #         gpt35_ablation = False
+    #     infer_objects(vlm_question_path, vlm_answer_path, interactable_object_path, gpt35_ablation)
+    # elif llm == 'llama2':
+    #     from approach.llm.llama import infer_objects
+    #     exit(0)
+    # elif llm.endswith('_pass'):
+    #     print(f'Passing LLM {llm} ...')
+    # else:
+    #     raise Exception('Unrecognized LLM!')
+
+
+
+
+
+
+    # STEP #2
+    # Open-vocabulary object detection
+    ovod_start_time = time.time()
+    logging.info(f'STEP #2 OVOD {ovod} analysis started at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    vlm_questions = {}
+    with open(complete_vlm_question_path, 'r') as q_file:
+        for line in q_file:
+            line_data = json.loads(line)
+            vlm_questions[line_data['question_id']] = line_data
+            # print(line_data['question_id'])
+
+    if ovod == 'grounding_dino':
+        from approach.ovod.GroundingDINO.demo.inference_on_a_image import process_grounding_dino
+
+        original_path = os.path.dirname(__file__)
+        os.chdir(grounding_dino_path)
+
+        # with open(llm_candidate_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        with open(interactable_object_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+            llm_candidate = json.load(llm_candidate_file)
+            all_oovd_res = {}
+            for image_index in llm_candidate.keys():
+                print(image_index)
+                 # TODO: 
+                if (image_index != '1160'):
+                    ovod_image_path = os.path.join(images_dir, vlm_questions[int(image_index)]["image"])
+                    ovod_candidates = llm_candidate[image_index]['interactable_objects']
+                    # grounding_dino_command = generate_grounding_dino_command(ovod_image_path, ovod_output_dir, ovod_candidates)
+                    # print(grounding_dino_command)
+                    # os.system(grounding_dino_command)
+
+                    gdino_res = process_grounding_dino(
+                        config_file='groundingdino/config/GroundingDINO_SwinT_OGC.py', 
+                        checkpoint_path='weights/groundingdino_swint_ogc.pth', 
+                        image_path=ovod_image_path, 
+                        ovod_candidates=ovod_candidates, 
+                        output_dir=ovod_output_dir, 
+                        box_threshold=0.3, 
+                        text_threshold=0.25, 
+                        token_spans=None, 
+                        cpu_only=False
+                    )
+                    # print(gdino_res)
+
+                    object_oovd_item = copy.deepcopy(llm_candidate[image_index])
+                    object_oovd_item['oovd_result'] = gdino_res
+                    print(object_oovd_item)
+                    all_oovd_res[image_index] = object_oovd_item
+
+            json.dump(all_oovd_res, oovd_file, indent=4)
+
+
+        os.chdir(original_path)
+
+    elif ovod == 'ape_d' or ovod == 'ape_d_abl':
+        from approach.ovod.APE.demo.ape_inference import run_ape_model_inference
+
+        original_path = os.path.dirname(__file__)
+        os.chdir(active_ape_path)
+
+        all_ape_res = []
+        reflection_traces = []
+        # General
+        # with open(vlm_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        # GPT-4 eval
+        with open(gpt4_results_answer_path, 'r') as llm_candidate_file:
+        # Gemini abl i eval
+        # with open(interactability_abl_results_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+
+            # llm_candidate = json.load(llm_candidate_file)
+            all_oovd_res = {}
+
+            candidate_lines = select_jsonl_lines(
+                llm_candidate_file.readlines(),
+                start_index=start_index,
+                end_index=end_index,
+                shard_index=shard_index,
+                num_shards=num_shards,
+            )
+
+            for line in tqdm(candidate_lines):
+                ans_item = {}
+
+                line_data = json.loads(line)
+
+            # for image_index in llm_candidate.keys():
+                # print(image_index)
+                 # TODO: 
+                # if (image_index != '1160'):
+                # image_index = line_data['question_id']
+
+                image_name = vlm_questions[line_data['question_id']]["image"]
+                ovod_image_path = os.path.join(images_dir, image_name)
+
+                # General
+                # try:
+                #     if line_data['text'].startswith(" ```json"):
+                #         ovod_candidates = json.loads(line_data['text'][8:-4])['objects']
+                #     elif line_data['text'].startswith(' {\"objects\"'):
+                #         ovod_candidates = json.loads(line_data['text'])['objects']
+                #     else:
+                #         print(f"Error for decoding IVO json for {image_name}.")
+                # except Exception as e:
+                #     print(f"Error for decoding IVO json for image {image_name}. Error: {e}")
+                #     continue
+
+
+                # GPT-4v
+                if line_data['text']:
+                    ovod_candidates = line_data['text']['objects']
+                else:
+                    continue
+                all_res = []
+                for ocd in ovod_candidates.keys():
+                    referring_expr_str = ovod_candidates[ocd]
+
+                    translator = str.maketrans('', '', string.punctuation)
+                    if type(referring_expr_str) is str:
+                        referring_expr_str = referring_expr_str.translate(translator)
+                    else:
+                        # dict
+                        referring_expr_str = ' '.join(referring_expr_str.values())
+                        referring_expr_str = referring_expr_str.translate(translator)
+                    referring_expr_str = f'{ocd}: {referring_expr_str}'
+                    # print(image_name, referring_expr_str)
+                    all_res.append(referring_expr_str)
+
+                if ovod == 'ape_d':
+                    ape_threshold = 0.15
+                elif ovod == 'ape_d_abl':
+                    ape_threshold = 0.1
+
+                ape_res = []
+                try:
+                    ape_res = run_ape_model_inference(
+                        config_file='configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py',
+                        input_path=ovod_image_path,
+                        # GPU 0
+                        output_path=f'./realfse_{vlm}_{ovod}_gpu0123',
+                        confidence_threshold=ape_threshold,
+                        text_prompt=', '.join(all_res),
+                        with_box=True,
+                        with_mask=False,
+                        with_sseg=False,
+                        opts=[
+                                f"train.init_checkpoint='{active_ape_checkpoint}'",
+                                "model.model_language.cache_dir=''",
+                                "model.model_vision.select_box_nums_for_evaluation=500",
+                                "model.model_vision.text_feature_bank_reset=True",
+                                "model.model_vision.backbone.net.xattn=False",
+                                "model.model_vision.transformer.encoder.pytorch_attn=True",
+                                "model.model_vision.transformer.decoder.pytorch_attn=True"
+                        ]
+                    )
+                except Exception as e:
+                    print(f"Error for image {image_name}. Error: {e}")
+
+                def reflection_detector(candidates, previous):
+                    if not candidates:
+                        return previous
+                    try:
+                        refined = run_ape_model_inference(
+                            config_file='configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py',
+                            input_path=ovod_image_path,
+                            output_path=f'./realfse_{vlm}_{ovod}_gpu0123',
+                            confidence_threshold=ape_threshold,
+                            text_prompt=', '.join(candidates),
+                            with_box=True,
+                            with_mask=False,
+                            with_sseg=False,
+                            opts=[
+                                    f"train.init_checkpoint='{active_ape_checkpoint}'",
+                                    "model.model_language.cache_dir=''",
+                                    "model.model_vision.select_box_nums_for_evaluation=500",
+                                    "model.model_vision.text_feature_bank_reset=True",
+                                    "model.model_vision.backbone.net.xattn=False",
+                                    "model.model_vision.transformer.encoder.pytorch_attn=True",
+                                    "model.model_vision.transformer.decoder.pytorch_attn=True"
+                            ]
+                        )
+                        return previous + refined
+                    except Exception as e:
+                        print(f"Error during reflection redetection for {image_name}. Error: {e}")
+                        return previous
+
+                reflection_result = run_optional_reflection(
+                    ovod_image_path,
+                    ape_res,
+                    detector=reflection_detector,
+                    enabled=enable_reflection,
+                    model_profile=reflection_profile,
+                    max_iterations=max_reflection_iterations,
+                )
+                if reflection_result is not None:
+                    ape_res = reflection_result["detections"]
+                    reflection_traces.append(
+                        {
+                            "image": image_name,
+                            "image_id": extract_image_id(image_name),
+                            "trace": reflection_result["trace"],
+                            "max_iterations_reached": reflection_result["max_iterations_reached"],
+                        }
+                    )
+
+                all_ape_res.extend(enrich_ape_results(ape_res, image_name, extract_image_id))
+                # object_oovd_item = copy.deepcopy(llm_candidate[image_index])
+                # object_oovd_item['ape_result'] = all_ape_res
+                # # print(object_oovd_item)
+                # all_oovd_res[image_index] = object_oovd_item
+
+            write_json_atomic(oovd_object_bbox_path, all_ape_res)
+            if enable_reflection:
+                write_json_atomic(f"{oovd_object_bbox_path}.reflection.json", reflection_traces)
+
+
+        os.chdir(original_path)                    
+
+    elif ovod.endswith('_pass'):
+        print(f'Passing OVOD {ovod} ...')
+    else:
+        raise Exception('Unrecognized OVOD!')
+
+    logging.info(f'STEP #2 OVOD {ovod} analysis completed at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    logging.info(f'STEP #2 OVOD {ovod} analysis total time taken: {time.time() - ovod_start_time} seconds')
+# icse & fse
+def extract_image_id(image_name):
+    # Split the image name into parts and form the image_id accordingly
+    parts = image_name.split('_')
+    print(image_name)
+    base, extension = parts[1].split('.')
+    return int(parts[0] + base.zfill(3))
+# icse_rebuttal
+# def extract_image_id(image_name):
+#     # Split the image name into parts by underscore
+#     parts = image_name.split('_')
+#     # Extracting the first part as the base and the numeric portion of the third part before the file extension
+#     base = parts[0]  # This will give '625470'
+#     numeric_part = parts[2].split('.')[0]  # This will give 'b3'
+#     # Removing non-numeric characters from 'b3'
+#     numeric_part = ''.join(filter(str.isdigit, numeric_part))
+#     # Zfill is used to ensure the numeric part has at least 3 digits, then combining with base
+#     print(int(base + numeric_part.zfill(3)))
+#     return int(base + numeric_part.zfill(3))
+
+
+def main():
+    vlms = ['llava7b', 'bing', 'gpt4v']
+    llms = ['llama2', 'gpt_3.5_turbo']
+    ovods = ['grounding_dino', 'glip', 'ape_d']
+
+
+# python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d
+# python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d_abl
+# python method.py -v claude35sonnet -l gpt_3.5_turbo -o ape_d
+# python method.py -v gemini -l gpt_3.5_turbo -o ape_d
+if __name__=='__main__':
+    # main()
+
+    parser = argparse.ArgumentParser("Method", add_help=True)
+    parser.add_argument("--vlm", "-v", type=str, required=True, help="vlm")
+    parser.add_argument("--llm", "-l", type=str, required=True, help="llm")
+    parser.add_argument("--ovod", "-o", type=str, required=True, help="ovod")
+    parser.add_argument("--start-index", type=int, default=None, help="first JSONL row to process")
+    parser.add_argument("--end-index", type=int, default=None, help="exclusive JSONL row end")
+    parser.add_argument("--shard-index", type=int, default=None, help="zero-based shard index")
+    parser.add_argument("--num-shards", type=int, default=None, help="total number of shards")
+    parser.add_argument("--enable-reflection", action="store_true", help="run PII.5/PII.6 advisor reflection loop")
+    parser.add_argument("--reflection-profile", default="default", help="model profile for the reflection advisor")
+    parser.add_argument("--max-reflection-iterations", type=int, default=10, help="max advisor reflection rounds")
+    parser.add_argument("--questions", dest="questions_path", help="question manifest JSONL")
+    parser.add_argument("--candidates", dest="candidates_path", help="candidate JSON/JSONL for the selected detector")
+    parser.add_argument("--images-dir", dest="images_path", help="directory containing XR screenshots")
+    parser.add_argument("--output", dest="output_path", help="prediction JSON path; selection suffixes are added automatically")
+    parser.add_argument("--ape-root", help="APE repository directory")
+    parser.add_argument("--ape-checkpoint", help="APE checkpoint path")
+    parser.add_argument("--log-file", help="optional log file; defaults to stderr")
+    args = parser.parse_args()
+
+    gpu = '1-3'
+    stage = '1'
+    idx = '0001'
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+        datefmt='%m/%d/%Y %H:%M:%S',
+        level=logging.INFO,
+        filename=args.log_file,
+        )
+
+    method(
+        args.vlm,
+        args.llm,
+        args.ovod,
+        start_index=args.start_index,
+        end_index=args.end_index,
+        shard_index=args.shard_index,
+        num_shards=args.num_shards,
+        enable_reflection=args.enable_reflection,
+        reflection_profile=args.reflection_profile,
+        max_reflection_iterations=args.max_reflection_iterations,
+        questions_path=args.questions_path,
+        candidates_path=args.candidates_path,
+        images_path=args.images_path,
+        output_path=args.output_path,
+        ape_root=args.ape_root,
+        ape_checkpoint=args.ape_checkpoint,
+    )
+
+# CUDA_VISIBLE_DEVICES=0 python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d > ../log/realfse/240910_gpt4v_2_g0_0001.txt
+# CUDA_VISIBLE_DEVICES=3 python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d_abl > ../log/realfse/240910_gpt4v_ape_d_abl_2_g3_0001.txt
+# CUDA_VISIBLE_DEVICES=3 python method.py -v gpt4v_abl -l gpt_3.5_turbo -o ape_d > ../log/realfse/240912_gpt4v_abl_1_g1-3_0001.txt

approach/method_claude.py

@@ -0,0 +1,656 @@
+import os
+import re
+import sys
+import json
+import time
+import random
+import string
+import argparse
+import pandas as pd
+import copy
+import logging
+from tqdm import tqdm
+import PIL
+from PIL import ImageFile
+PIL.ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(BASE_DIR)
+from approach.pipeline_utils import (
+    enrich_ape_results,
+    output_path_for_selection,
+    select_jsonl_lines,
+    write_json_atomic,
+)
+
+os.environ['TORCH_HOME'] = './.cache'
+os.environ['HF_HOME'] = './.cache'
+
+
+vlm = 'llava7b'
+llm = 'gpt_3.5_turbo'
+ovod = 'grounding_dino'
+
+vlm_prompt = ''
+
+### Configurable
+host_device = os.getenv('ORIENTER_LEGACY_HOST', 'local')
+perspective = 'all_perspective'
+# perspective = 'direct_back'
+# perspective = 'direct_front'
+# perspective = 'direct_side'
+# perspective = 'direct_top'
+# perspective = 'eyelevel'
+# perspective = 'overlook'
+### Configurable
+
+llava_path = os.path.join(BASE_DIR, 'approach/vlm/LLaVA/llava/eval')
+grounding_dino_path = os.path.join(BASE_DIR, 'approach/ovod/GroundingDINO')
+ape_path = os.path.join(BASE_DIR, 'approach/ovod/APE')
+# TODO: data on the CUHK server
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_merged/images/interactable')
+# icse
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_merged/images/test_set_merged')
+# icse_rebuttal
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_angleview/images/all_perspective')
+# fse
+images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_det/images/union3')
+
+
+# def generate_grounding_dino_command(ovod_image_path, ovod_output_dir, ovod_candidates):
+#     gdino_object_str = ' . '.join(ovod_candidates)
+
+#     gdino_command = f'''
+#     python demo/inference_on_a_image.py \
+#         -c groundingdino/config/GroundingDINO_SwinT_OGC.py \
+#         -p weights/groundingdino_swint_ogc.pth \
+#         -i {ovod_image_path} \
+#         -o "{ovod_output_dir}" \
+#         -t "{gdino_object_str}"
+#     '''
+#     return gdino_command
+
+
+def generate_question_file(img_folder, dst_path):
+    # Ensure the directory exists
+    if not os.path.exists(img_folder):
+        print(f"Error: Directory {img_folder} does not exist.")
+        return
+
+    # List all files in the directory
+    all_files = os.listdir(img_folder)
+
+    # Filter out files that are not images (based on extension). You can add more if needed.
+    image_extensions = [".jpg", ".jpeg", ".png", ".bmp", ".gif", ".tiff"]
+    image_files = [f for f in all_files if any(f.lower().endswith(ext) for ext in image_extensions)]
+
+    # Open the output file for writing
+    with open(dst_path, 'w') as out_file:
+        for index, img_file in enumerate(image_files):
+            data = {
+                "question_id": index,
+                "image": img_file,
+                "text": vlm_prompt,
+                "category": "detail"
+            }
+            out_file.write(json.dumps(data) + '\n')
+
+    print(f"Processed {len(image_files)} images. Output saved to {dst_path}.")
+
+
+def generate_answer_id(length=20):
+    # Define the characters that can be used in the string
+    characters = string.ascii_letters + string.digits
+    # Generate a random string of the specified length
+    answer_id = ''.join(random.choice(characters) for _ in range(length))
+    return answer_id
+
+
+def method(vlm=vlm, llm=llm, ovod=ovod, start_index=None, end_index=None, shard_index=None, num_shards=None):
+    # vlm_question_path = os.path.join(BASE_DIR, 'approach/vlm/sampled_vlm_questions.jsonl')
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_vlm_questions_from671.jsonl')
+    # icse
+    # complete_vlm_question_abl_i_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_complete_vlm_questions_ablation_interactability.jsonl')
+
+    # icse
+    # complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_complete_vlm_questions.jsonl')
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/gpt4v_failure_questions.jsonl')
+    # icse_rebuttal
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_vlm_questions.jsonl')
+    # complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_complete_vlm_questions.jsonl')
+    # fse
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/fse_union3_{vlm}_questions.jsonl')
+    complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_union3_complete_{vlm}_questions.jsonl')
+    vlm_question_path = complete_vlm_question_path
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/gpt4v_failure_questions.jsonl')
+    # playground/data/coco2014_val_qa_eval/qa90_questions.jsonl
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_slowerspeed_1.jsonl')
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_higherspeed_r3_1_from195.jsonl')
+
+    # icse
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/cuhk_icse_test_set_merged_{vlm}_answer.jsonl')
+    # llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer.jsonl')
+    # gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_ablation_no_interactability_r1_0.jsonl')
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_cuhk_icse_test_set_merged_gpt4v_answer_higherspeed_r3_0.jsonl')
+    # interactability_abl_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_hit_icse_test_set_merged_gemini_answer_ablation_no_interactability_r1_0.jsonl')
+    # icse_rebuttal
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # fse
+    vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_uninon3_{vlm}_answer.jsonl')
+    llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_fse_uninon3_{vlm}_answer.jsonl')
+    gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_union3_{vlm}_answer.jsonl')
+
+    # icse
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_icse_rbt_{perspective}_gpt4v_answer.jsonl')
+    # icse_rebuttal
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_{vlm}_answer_265_528.jsonl')
+    # fse
+    gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_uninon3_gpt4v_answer.jsonl')
+    # print(gpt4_results_answer_path)
+    # interactability_abl_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_icse_rbt_{perspective}_answer.jsonl')
+
+    # icse
+    # # /path/to/answer-file-our.jsonl
+    # # llm_candidate_path = os.path.join(BASE_DIR, f'approach/llm/{vlm}_{llm}_cancidate_objects.json')
+    # interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/icse_test_set_merged_{vlm}_{llm}_interactable_objects.json')
+    # ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/icse_test_set_merged_{ovod}')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_object_bbox_gpu3_2.json')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_interactability_object_bbox_gpu0.json')
+    # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_feedback_object_bbox_gpu2.json')
+
+    # icse_rebuttal
+    # interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/icse_rbt_{perspective}_{vlm}_{llm}_interactable_objects.json')
+    # ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/icse_rbt_{perspective}_{ovod}')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_object_bbox_gpu3_2.json')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_interactability_object_bbox_gpu0.json')
+    # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_rbt_{perspective}_{vlm}_{llm}_{ovod}_object_bbox_gpu3_265_528.json')
+
+    gpu = '2'
+    # fse
+    interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/realfse_union3_{vlm}_{llm}_interactable_objects.json') # TODO: Tentatively unused
+    ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/realfse_union3_{ovod}')
+    # GPU 0
+    oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/realfse_union3_{vlm}_{ovod}_object_bbox_gpu{gpu}.json')
+    oovd_object_bbox_path = output_path_for_selection(
+        oovd_object_bbox_path,
+        start_index=start_index,
+        end_index=end_index,
+        shard_index=shard_index,
+        num_shards=num_shards,
+    )
+
+    llava7b_model_path = os.path.join(BASE_DIR, 'approach/vlm/LLaVA/checkpoints/llava-v1.5-7b')
+    llava7b_cmd = f'''
+    python model_vqa.py \
+        --model-path {llava7b_model_path} \
+        --question-file \
+        {complete_vlm_question_path} \
+        --image-folder \
+        {images_dir} \
+        --answers-file \
+        {llava_vlm_answer_path}
+    '''
+
+    # STEP #0
+    # First time of running
+    # TODO: Check whether it is first-time running
+    # generate_question_file(images_dir, vlm_question_path)
+
+    # STEP #1
+    # VLM - Get local context
+    # print(f'VLM {vlm} analysis begins ...')
+    # vlm_start_time = time.time()
+    # logging.info(f'STEP #1 VLM {vlm} analysis started at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    # if vlm == 'llava7b':
+    #     original_path = os.path.dirname(__file__)
+    #     os.chdir(llava_path)
+    #     os.system(llava7b_cmd)
+    #     os.chdir(original_path)
+    # elif vlm == 'generate_q_file':
+    #     from approach.vlm.gpt4v.gpt4v import process_image_q
+    #     with open(vlm_question_path, 'r') as q_file, open(complete_vlm_question_path, 'w') as cq_file:  # Open answer file in append mode
+    #         qfile_lines = q_file.readlines()
+    #         key_idx = 0
+    #         for line in tqdm(qfile_lines):
+    #             key_idx = (key_idx + 1) % 4
+
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             if vlm == 'gpt4v_abl':
+    #                 gpt4v_ablation = True
+    #             elif vlm == 'gpt4v':
+    #                 gpt4v_ablation = False
+    #             gpt4v_q = process_image_q(image_question, image_path, key_idx) 
+    #             data = {
+    #                 "question_id": line_data['question_id'],
+    #                 "image": line_data['image'],
+    #                 "text": gpt4v_q,
+    #                 "category": "detail"
+    #             }
+    #             cq_file.write(json.dumps(data) + '\n')       
+    # elif vlm == 'gpt4v' or vlm == 'gpt4v_abl' or vlm == 'claude35sonnet' or vlm == 'gemini15pro':
+    #     from approach.vlm.gpt4v.gpt4v import process_image
+    #     with open(vlm_question_path, 'r') as q_file, open(vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+    #         qfile_lines = q_file.readlines()
+    #         key_idx = 0
+    #         for line in tqdm(qfile_lines):
+    #             key_idx = (key_idx + 1) % 4
+
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             gpt4v_ablation = False
+    #             if vlm == 'gpt4v_abl':
+    #                 gpt4v_ablation = True
+    #             elif vlm == 'gpt4v':
+    #                 gpt4v_ablation = False
+    #             gpt4v_res = process_image(vlm, image_question, image_path, gpt4v_ablation, key_idx)
+    #             print(gpt4v_res)
+    #             # time.sleep(3)
+
+    #             ans_item = {
+    #                 "question_id": line_data['question_id'], 
+    #                 "prompt": '', 
+    #                 "text": gpt4v_res, 
+    #                 "answer_id": generate_answer_id(), 
+    #                 "model_id": vlm, 
+    #                 "metadata": {}
+    #             }
+
+    #             a_file.write(json.dumps(ans_item) + '\n')  
+    #             a_file.flush()
+    # elif vlm == 'bing':
+    #     from approach.vlm.bing.bing import context_conversation
+    #     with open(vlm_question_path, 'r') as q_file, open(vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+    #         for line in tqdm(q_file):
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             bing_res = context_conversation(image_question, image_path)
+    #             print(bing_res)
+    #             time.sleep(15)
+
+    #             ans_item = {
+    #                 "question_id": line_data['question_id'], 
+    #                 "prompt": image_question, 
+    #                 "text": bing_res, 
+    #                 "answer_id": generate_answer_id(), 
+    #                 "model_id": vlm, 
+    #                 "metadata": {}
+    #             }
+
+    #             a_file.write(json.dumps(ans_item) + '\n')  
+    # elif vlm == 'gemini':
+    #     import pathlib
+    #     import textwrap
+
+    #     import google.generativeai as genai
+
+    #     GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "[GOOGLE_API_KEY]")
+    #     genai.configure(api_key=GOOGLE_API_KEY)
+
+    #     safety_settings = [
+    #         {
+    #             "category": "HARM_CATEGORY_DANGEROUS",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_HARASSMENT",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_HATE_SPEECH",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #         {
+    #             "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+    #             "threshold": "BLOCK_NONE",
+    #         },
+    #     ]
+
+    #     # before fse
+    #     # model = genai.GenerativeModel('gemini-pro-vision')
+    #     # fse
+    #     model = genai.GenerativeModel('gemini-1.5-pro')
+
+    #     with open(vlm_question_path, 'r') as q_file, open(gemini_vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+    #         qfile_lines = q_file.readlines()[0:2]
+    #         for line in tqdm(qfile_lines):
+
+    #             ans_item = {}
+
+    #             line_data = json.loads(line)
+    #             image_path = os.path.join(images_dir, line_data['image'])
+    #             image_question = line_data['text']
+
+    #             gemini_response = model.generate_content([image_question, PIL.Image.open(image_path)], safety_settings=safety_settings)
+    #             gemini_response.resolve()
+    #             try:
+    #                 gemini_response = gemini_response.text
+    #                 ans_item = {
+    #                     "question_id": line_data['question_id'], 
+    #                     "prompt": image_question, 
+    #                     "text": gemini_response, 
+    #                     "answer_id": generate_answer_id(), 
+    #                     "model_id": vlm, 
+    #                     "metadata": {}
+    #                 }
+
+    #                 a_file.write(json.dumps(ans_item) + '\n')  
+    #                 a_file.flush()
+    #             except Exception as e:
+    #                 print(f"Error for image {image_path}. Error: {e}")                
+    #             # print(gemini_response)
+    #             time.sleep(30)
+
+
+
+    # elif vlm.endswith('_pass'):
+    #     print(f'Passing VLM {vlm} ...')
+    # else:
+    #     raise Exception('Unrecognized VLM!')
+
+    # logging.info(f'STEP #1 VLM {vlm} analysis completed at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    # logging.info(f'STEP #1 VLM {vlm} analysis total time taken: {time.time() - vlm_start_time} seconds')
+
+
+
+
+
+    # Get label candidates
+    # if llm == 'gpt_3.5_turbo':
+    #     from approach.llm.gpt_polling import infer_object_candidates
+    # elif llm == 'llama2':
+    #     from approach.llm.llama import infer_object_candidates
+
+
+    # STEP #3
+    # if llm == 'gpt_3.5_turbo' or llm == 'gpt_3.5_turbo_abl':
+    #     from approach.llm.gpt_polling import infer_objects
+
+    #     if llm == 'gpt_3.5_turbo_abl':
+    #         gpt35_ablation = True
+    #     elif llm == 'gpt_3.5_turbo':
+    #         gpt35_ablation = False
+    #     infer_objects(vlm_question_path, vlm_answer_path, interactable_object_path, gpt35_ablation)
+    # elif llm == 'llama2':
+    #     from approach.llm.llama import infer_objects
+    #     exit(0)
+    # elif llm.endswith('_pass'):
+    #     print(f'Passing LLM {llm} ...')
+    # else:
+    #     raise Exception('Unrecognized LLM!')
+
+
+
+
+
+
+    # STEP #2
+    # Open-vocabulary object detection
+    ovod_start_time = time.time()
+    logging.info(f'STEP #2 OVOD {ovod} analysis started at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    vlm_questions = {}
+    with open(complete_vlm_question_path, 'r') as q_file:
+        for line in q_file:
+            line_data = json.loads(line)
+            vlm_questions[line_data['question_id']] = line_data
+            # print(line_data['question_id'])
+
+    if ovod == 'grounding_dino':
+        from approach.ovod.GroundingDINO.demo.inference_on_a_image import process_grounding_dino
+
+        original_path = os.path.dirname(__file__)
+        os.chdir(grounding_dino_path)
+
+        # with open(llm_candidate_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        with open(interactable_object_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+            llm_candidate = json.load(llm_candidate_file)
+            all_oovd_res = {}
+            for image_index in llm_candidate.keys():
+                print(image_index)
+                 # TODO: 
+                if (image_index != '1160'):
+                    ovod_image_path = os.path.join(images_dir, vlm_questions[int(image_index)]["image"])
+                    ovod_candidates = llm_candidate[image_index]['interactable_objects']
+                    # grounding_dino_command = generate_grounding_dino_command(ovod_image_path, ovod_output_dir, ovod_candidates)
+                    # print(grounding_dino_command)
+                    # os.system(grounding_dino_command)
+
+                    gdino_res = process_grounding_dino(
+                        config_file='groundingdino/config/GroundingDINO_SwinT_OGC.py', 
+                        checkpoint_path='weights/groundingdino_swint_ogc.pth', 
+                        image_path=ovod_image_path, 
+                        ovod_candidates=ovod_candidates, 
+                        output_dir=ovod_output_dir, 
+                        box_threshold=0.3, 
+                        text_threshold=0.25, 
+                        token_spans=None, 
+                        cpu_only=False
+                    )
+                    # print(gdino_res)
+
+                    object_oovd_item = copy.deepcopy(llm_candidate[image_index])
+                    object_oovd_item['oovd_result'] = gdino_res
+                    print(object_oovd_item)
+                    all_oovd_res[image_index] = object_oovd_item
+
+            json.dump(all_oovd_res, oovd_file, indent=4)
+
+
+        os.chdir(original_path)
+
+    elif ovod == 'ape_d' or ovod == 'ape_d_abl':
+        from approach.ovod.APE.demo.ape_inference import run_ape_model_inference
+
+        original_path = os.path.dirname(__file__)
+        os.chdir(ape_path)
+
+        # with open(llm_candidate_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+
+        all_ape_res = []
+        # General
+        with open(vlm_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        # GPT-4 eval
+        # with open(gpt4_results_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        # Gemini abl i eval
+        # with open(interactability_abl_results_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+
+            # llm_candidate = json.load(llm_candidate_file)
+            all_oovd_res = {}
+
+
+            # candidate_lines = llm_candidate_file.readlines()[265:528]
+            # GPU 0
+            # candidate_lines = llm_candidate_file.readlines()[0:230]
+            # candidate_lines = llm_candidate_file.readlines()[230:460]
+            candidate_lines = select_jsonl_lines(
+                llm_candidate_file.readlines(),
+                start_index=start_index,
+                end_index=end_index,
+                shard_index=shard_index,
+                num_shards=num_shards,
+            )
+            # candidate_lines = llm_candidate_file.readlines()[690:919]
+            # candidate_lines = llm_candidate_file.readlines()
+            for line in tqdm(candidate_lines):
+                ans_item = {}
+
+                line_data = json.loads(line)
+
+            # for image_index in llm_candidate.keys():
+                # print(image_index)
+                 # TODO: 
+                # if (image_index != '1160'):
+                # image_index = line_data['question_id']
+
+                image_name = vlm_questions[line_data['question_id']]["image"]
+                ovod_image_path = os.path.join(images_dir, image_name)
+
+                # General
+                # try:
+                #     if line_data['text'].startswith(" ```json"):
+                #         ovod_candidates = json.loads(line_data['text'][8:-4])['objects']
+                #     elif line_data['text'].startswith(' {\"objects\"'):
+                #         ovod_candidates = json.loads(line_data['text'])['objects']
+                #     else:
+                #         print(f"Error for decoding IVO json for {image_name}.")
+                # except Exception as e:
+                #     print(f"Error for decoding IVO json for image {image_name}. Error: {e}")
+                #     continue
+
+
+                # GPT-4v
+                if line_data['text']:
+                    ovod_candidates = line_data['text']['objects']
+                else:
+                    continue
+
+
+                all_res = []
+                for ocd in ovod_candidates.keys():
+                    referring_expr_str = ovod_candidates[ocd]
+
+                    translator = str.maketrans('', '', string.punctuation)
+                    if type(referring_expr_str) is str:
+                        referring_expr_str = referring_expr_str.translate(translator)
+                    else:
+                        # dict
+                        referring_expr_str = ' '.join(referring_expr_str.values())
+                        referring_expr_str = referring_expr_str.translate(translator)
+                    referring_expr_str = f'{ocd}: {referring_expr_str}'
+                    # print(image_name, referring_expr_str)
+                    all_res.append(referring_expr_str)
+
+                if ovod == 'ape_d':
+                    ape_threshold = 0.15
+                elif ovod == 'ape_d_abl':
+                    ape_threshold = 0.1
+
+                ape_res = []
+                try:
+                    ape_res = run_ape_model_inference(
+                        config_file='configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py',
+                        input_path=ovod_image_path,
+                        # GPU 0
+                        output_path=f'./realfse_{vlm}_{ovod}_gpu0123',
+                        confidence_threshold=ape_threshold,
+                        text_prompt=', '.join(all_res),
+                        with_box=True,
+                        with_mask=False,
+                        with_sseg=False,
+                        opts=[
+                                "train.init_checkpoint='./ape_d_model_final.pth'",
+                                "model.model_language.cache_dir=''",
+                                "model.model_vision.select_box_nums_for_evaluation=500",
+                                "model.model_vision.text_feature_bank_reset=True",
+                                "model.model_vision.backbone.net.xattn=False",
+                                "model.model_vision.transformer.encoder.pytorch_attn=True",
+                                "model.model_vision.transformer.decoder.pytorch_attn=True"
+                        ]
+                    )
+                except Exception as e:
+                    print(f"Error for image {image_name}. Error: {e}")
+
+                all_ape_res.extend(enrich_ape_results(ape_res, image_name, extract_image_id))
+                # object_oovd_item = copy.deepcopy(llm_candidate[image_index])
+                # object_oovd_item['ape_result'] = all_ape_res
+                # # print(object_oovd_item)
+                # all_oovd_res[image_index] = object_oovd_item
+
+            write_json_atomic(oovd_object_bbox_path, all_ape_res)
+
+
+        os.chdir(original_path)                    
+
+    elif ovod.endswith('_pass'):
+        print(f'Passing OVOD {ovod} ...')
+    else:
+        raise Exception('Unrecognized OVOD!')
+
+    logging.info(f'STEP #2 OVOD {ovod} analysis completed at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    logging.info(f'STEP #2 OVOD {ovod} analysis total time taken: {time.time() - ovod_start_time} seconds')
+
+# icse & fse
+def extract_image_id(image_name):
+    # Split the image name into parts and form the image_id accordingly
+    parts = image_name.split('_')
+    print(image_name)
+    base, extension = parts[1].split('.')
+    return int(parts[0] + base.zfill(3))
+# icse_rebuttal
+# def extract_image_id(image_name):
+#     # Split the image name into parts by underscore
+#     parts = image_name.split('_')
+#     # Extracting the first part as the base and the numeric portion of the third part before the file extension
+#     base = parts[0]  # This will give '625470'
+#     numeric_part = parts[2].split('.')[0]  # This will give 'b3'
+#     # Removing non-numeric characters from 'b3'
+#     numeric_part = ''.join(filter(str.isdigit, numeric_part))
+#     # Zfill is used to ensure the numeric part has at least 3 digits, then combining with base
+#     print(int(base + numeric_part.zfill(3)))
+#     return int(base + numeric_part.zfill(3))
+
+
+def main():
+    vlms = ['llava7b', 'bing', 'gpt4v']
+    llms = ['llama2', 'gpt_3.5_turbo']
+    ovods = ['grounding_dino', 'glip', 'ape_d']
+
+
+# python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d
+# python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d_abl
+# python method.py -v claude35sonnet -l gpt_3.5_turbo -o ape_d
+# python method.py -v gemini -l gpt_3.5_turbo -o ape_d
+if __name__=='__main__':
+    # main()
+
+    parser = argparse.ArgumentParser("Method", add_help=True)
+    parser.add_argument("--vlm", "-v", type=str, required=True, help="vlm")
+    parser.add_argument("--llm", "-l", type=str, required=True, help="llm")
+    parser.add_argument("--ovod", "-o", type=str, required=True, help="ovod")
+    parser.add_argument("--start-index", type=int, default=None, help="first JSONL row to process")
+    parser.add_argument("--end-index", type=int, default=None, help="exclusive JSONL row end")
+    parser.add_argument("--shard-index", type=int, default=None, help="zero-based shard index")
+    parser.add_argument("--num-shards", type=int, default=None, help="total number of shards")
+    args = parser.parse_args()
+
+    gpu = '2'
+    stage = '2'
+    idx = '0001'
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+        datefmt='%m/%d/%Y %H:%M:%S',
+        level=logging.INFO,
+        # gpu
+        filename=f'../log/realfse/240910_{args.vlm}_{stage}_g{gpu}_{idx}.log'
+        )
+
+    method(
+        args.vlm,
+        args.llm,
+        args.ovod,
+        start_index=args.start_index,
+        end_index=args.end_index,
+        shard_index=args.shard_index,
+        num_shards=args.num_shards,
+    )
+
+# CUDA_VISIBLE_DEVICES=0 python method.py -v gpt4v -l gpt_3.5_turbo -o ape_d > ../log/realfse/240910_gpt4v_2_g0_0001.txt
+# CUDA_VISIBLE_DEVICES=3 python method_gemini.py -v gemini15pro -l gpt_3.5_turbo -o ape_d > ../log/realfse/240910_gemini15pro_2_g3_0001.txt
+# CUDA_VISIBLE_DEVICES=2 python method_claude.py -v claude35sonnet -l gpt_3.5_turbo -o ape_d > ../log/realfse/240910_claude35sonnet_2_g2_0001.txt

approach/method_fastuse.py

@@ -0,0 +1,736 @@
+import os
+import sys
+import json
+import time
+import random
+import string
+import argparse
+import copy
+import logging
+from tqdm import tqdm
+import PIL
+from PIL import ImageFile
+PIL.ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(BASE_DIR)
+from approach.pipeline_utils import (
+    enrich_ape_results,
+    output_path_for_selection,
+    run_optional_reflection,
+    select_jsonl_lines,
+    write_json_atomic,
+)
+
+os.environ['TORCH_HOME'] = './.cache'
+os.environ['HF_HOME'] = './.cache'
+
+
+vlm = 'llava7b'
+llm = 'gemini-3-flash-preview-nothinking'
+ovod = 'ape_d'
+
+vlm_prompt = ''
+
+### Configurable
+host_device = os.getenv('ORIENTER_LEGACY_HOST', 'local')
+perspective = 'all_perspective'
+# perspective = 'direct_back'
+# perspective = 'direct_front'
+# perspective = 'direct_side'
+# perspective = 'direct_top'
+# perspective = 'eyelevel'
+# perspective = 'overlook'
+### Configurable
+
+llava_path = os.path.join(BASE_DIR, 'approach/vlm/LLaVA/llava/eval')
+grounding_dino_path = os.path.join(BASE_DIR, 'approach/ovod/GroundingDINO')
+ape_path = os.path.join(BASE_DIR, 'approach/ovod/APE')
+# TODO: data on the CUHK server
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_merged/images/interactable')
+# icse
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_merged/images/test_set_merged')
+# icse_rebuttal
+# images_dir = os.path.join(BASE_DIR, 'dataset/data/coco_angleview/images/all_perspective')
+# fse
+images_dir = os.path.join(BASE_DIR, 'fastimg')
+
+
+# def generate_grounding_dino_command(ovod_image_path, ovod_output_dir, ovod_candidates):
+#     gdino_object_str = ' . '.join(ovod_candidates)
+
+#     gdino_command = f'''
+#     python demo/inference_on_a_image.py \
+#         -c groundingdino/config/GroundingDINO_SwinT_OGC.py \
+#         -p weights/groundingdino_swint_ogc.pth \
+#         -i {ovod_image_path} \
+#         -o "{ovod_output_dir}" \
+#         -t "{gdino_object_str}"
+#     '''
+#     return gdino_command
+
+
+def generate_question_file(img_folder, dst_path):
+    # Ensure the directory exists
+    if not os.path.exists(img_folder):
+        print(f"Error: Directory {img_folder} does not exist.")
+        return
+
+    # List all files in the directory
+    all_files = os.listdir(img_folder)
+
+    # Filter out files that are not images (based on extension). You can add more if needed.
+    image_extensions = [".jpg", ".jpeg", ".png", ".bmp", ".gif", ".tiff"]
+    image_files = [f for f in all_files if any(f.lower().endswith(ext) for ext in image_extensions)]
+
+    # Open the output file for writing
+    with open(dst_path, 'w') as out_file:
+        for index, img_file in enumerate(image_files):
+            data = {
+                "question_id": index,
+                "image": img_file,
+                "text": vlm_prompt,
+                "category": "detail"
+            }
+            out_file.write(json.dumps(data) + '\n')
+
+    print(f"Processed {len(image_files)} images. Output saved to {dst_path}.")
+
+
+def generate_answer_id(length=20):
+    # Define the characters that can be used in the string
+    characters = string.ascii_letters + string.digits
+    # Generate a random string of the specified length
+    answer_id = ''.join(random.choice(characters) for _ in range(length))
+    return answer_id
+
+
+def method(
+    vlm=vlm,
+    llm=llm,
+    ovod=ovod,
+    start_index=None,
+    end_index=None,
+    shard_index=None,
+    num_shards=None,
+    enable_reflection=False,
+    reflection_profile="default",
+    max_reflection_iterations=10,
+    questions_path=None,
+    candidates_path=None,
+    images_path=None,
+    output_path=None,
+    ape_root=None,
+    ape_checkpoint=None,
+):
+    images_dir = images_path or globals()["images_dir"]
+    active_ape_path = ape_root or globals()["ape_path"]
+    active_ape_checkpoint = ape_checkpoint or "./ape_d_model_final.pth"
+    # vlm_question_path = os.path.join(BASE_DIR, 'approach/vlm/sampled_vlm_questions.jsonl')
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_vlm_questions_from671.jsonl')
+    # icse
+    # complete_vlm_question_abl_i_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_complete_vlm_questions_ablation_interactability.jsonl')
+
+    # icse
+    # complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_test_set_merged_complete_vlm_questions.jsonl')
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/gpt4v_failure_questions.jsonl')
+    # icse_rebuttal
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_vlm_questions.jsonl')
+    # complete_vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_complete_vlm_questions.jsonl')
+    # fse
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/fse_fastuse_{vlm}_questions.jsonl')
+    complete_vlm_question_path = questions_path or os.path.join(BASE_DIR, f'approach/vlm/realfse_fastuse_complete_{vlm}_questions.jsonl')
+    vlm_question_path = complete_vlm_question_path
+    # vlm_question_path = os.path.join(BASE_DIR, f'approach/vlm/gpt4v_failure_questions.jsonl')
+    # playground/data/coco2014_val_qa_eval/qa90_questions.jsonl
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_slowerspeed_1.jsonl')
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_higherspeed_r3_1_from195.jsonl')
+
+    # icse
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/cuhk_icse_test_set_merged_{vlm}_answer.jsonl')
+    # llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer.jsonl')
+    # gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_test_set_merged_{vlm}_answer_ablation_no_interactability_r1_0.jsonl')
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_cuhk_icse_test_set_merged_gpt4v_answer_higherspeed_r3_0.jsonl')
+    # interactability_abl_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_hit_icse_test_set_merged_gemini_answer_ablation_no_interactability_r1_0.jsonl')
+    # icse_rebuttal
+    # vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_icse_rbt_{perspective}_{vlm}_answer.jsonl')
+    # fse
+    vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_fastuse_{vlm}_answer.jsonl')
+    llava_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/{host_device}_fse_fastuse_{vlm}_answer.jsonl')
+    gemini_vlm_answer_path = os.path.join(BASE_DIR, f'approach/vlm/realfse_fastuse_{vlm}_answer.jsonl')
+
+    # icse
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_icse_rbt_{perspective}_gpt4v_answer.jsonl')
+    # icse_rebuttal
+    # gpt4_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/icse_rbt_{perspective}_{vlm}_answer_265_528.jsonl')
+    # fse
+    gpt4_results_answer_path = candidates_path or os.path.join(BASE_DIR, f'approach/vlm/realfse_fastuse_gpt4v_answer.jsonl')
+    # print(gpt4_results_answer_path)
+    # interactability_abl_results_answer_path = os.path.join(BASE_DIR, f'approach/vlm/aaa_icse_rbt_{perspective}_answer.jsonl')
+
+    # icse
+    # # /path/to/answer-file-our.jsonl
+    # # llm_candidate_path = os.path.join(BASE_DIR, f'approach/llm/{vlm}_{llm}_cancidate_objects.json')
+    # interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/icse_test_set_merged_{vlm}_{llm}_interactable_objects.json')
+    # ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/icse_test_set_merged_{ovod}')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_object_bbox_gpu3_2.json')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_interactability_object_bbox_gpu0.json')
+    # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_feedback_object_bbox_gpu2.json')
+
+    # icse_rebuttal
+    # interactable_object_path = os.path.join(BASE_DIR, f'approach/llm/icse_rbt_{perspective}_{vlm}_{llm}_interactable_objects.json')
+    # ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/icse_rbt_{perspective}_{ovod}')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_object_bbox_gpu3_2.json')
+    # # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_test_set_merged_{vlm}_{llm}_{ovod}_ablation_no_interactability_object_bbox_gpu0.json')
+    # oovd_object_bbox_path = os.path.join(BASE_DIR, f'approach/ovod/icse_rbt_{perspective}_{vlm}_{llm}_{ovod}_object_bbox_gpu3_265_528.json')
+
+    gemini31_results_answer_path = candidates_path or os.path.join(BASE_DIR, f'approach/vlm/realfse_fastuse_gemini31pro_answer.jsonl')
+    gpu = '1-3'
+    # fse
+    interactable_object_path = candidates_path or os.path.join(BASE_DIR, f'approach/llm/realfse_fastuse_{vlm}_{llm}_interactable_objects.json') # TODO: Tentatively unused
+    ovod_output_dir = os.path.join(BASE_DIR, f'approach/ovod/output/realfse_fastuse_{ovod}')
+    # GPU 0
+    base_output_path = output_path or os.path.join(BASE_DIR, f'approach/ovod/realfse_fastuse_{vlm}_{ovod}_object_bbox_gpu{gpu}.json')
+    oovd_object_bbox_path = output_path_for_selection(
+        base_output_path,
+        start_index=start_index,
+        end_index=end_index,
+        shard_index=shard_index,
+        num_shards=num_shards,
+    )
+
+    os.makedirs(ovod_output_dir, exist_ok=True)
+    llava7b_model_path = os.path.join(BASE_DIR, 'approach/vlm/LLaVA/checkpoints/llava-v1.5-7b')
+    llava7b_cmd = f'''
+    python model_vqa.py \
+        --model-path {llava7b_model_path} \
+        --question-file \
+        {complete_vlm_question_path} \
+        --image-folder \
+        {images_dir} \
+        --answers-file \
+        {llava_vlm_answer_path}
+    '''
+
+    # # STEP #0
+    # # First time of running
+    # # TODO: Check whether it is first-time running
+    generate_question_file(images_dir, vlm_question_path)
+
+    # STEP #1
+    # VLM - Get local context
+    print(f'VLM {vlm} analysis begins ...')
+    vlm_start_time = time.time()
+    logging.info(f'STEP #1 VLM {vlm} analysis started at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    if vlm == 'llava7b':
+        original_path = os.path.dirname(__file__)
+        os.chdir(llava_path)
+        os.system(llava7b_cmd)
+        os.chdir(original_path)
+    elif vlm == 'generate_q_file':
+        from approach.vlm.gpt4v.gpt4v import process_image_q
+        with open(vlm_question_path, 'r') as q_file, open(complete_vlm_question_path, 'w') as cq_file:  # Open answer file in append mode
+            qfile_lines = q_file.readlines()
+            key_idx = 0
+            for line in tqdm(qfile_lines):
+                key_idx = (key_idx + 1) % 4
+
+                ans_item = {}
+
+                line_data = json.loads(line)
+                image_path = os.path.join(images_dir, line_data['image'])
+                image_question = line_data['text']
+
+                if vlm == 'gpt4v_abl':
+                    gpt4v_ablation = True
+                elif vlm == 'gpt4v':
+                    gpt4v_ablation = False
+                gpt4v_q = process_image_q(image_question, image_path, key_idx)
+                data = {
+                    "question_id": line_data['question_id'],
+                    "image": line_data['image'],
+                    "text": gpt4v_q,
+                    "category": "detail"
+                }
+                cq_file.write(json.dumps(data) + '\n')
+    elif vlm == 'gpt4v' or vlm == 'gpt4v_abl' or vlm == 'claude35sonnet' or vlm == 'gemini15pro' or vlm == 'gemini31pro':
+        from approach.vlm.gpt4v.gpt4v import process_image
+        with open(vlm_question_path, 'r') as q_file, open(vlm_answer_path, 'w') as a_file:  # Open answer file in append mode
+            qfile_lines = q_file.readlines()
+            key_idx = 0
+            for line in tqdm(qfile_lines):
+                key_idx = (key_idx + 1) % 4
+
+                ans_item = {}
+
+                line_data = json.loads(line)
+                image_path = os.path.join(images_dir, line_data['image'])
+                image_question = line_data['text']
+
+                gpt4v_ablation = False
+                if vlm == 'gpt4v_abl':
+                    gpt4v_ablation = True
+                elif vlm == 'gpt4v':
+                    gpt4v_ablation = False
+                gpt4v_res = process_image(vlm, image_question, image_path, gpt4v_ablation, key_idx)
+                print(gpt4v_res)
+                # time.sleep(3)
+
+                ans_item = {
+                    "question_id": line_data['question_id'],
+                    "prompt": '',
+                    "text": gpt4v_res,
+                    "answer_id": generate_answer_id(),
+                    "model_id": vlm,
+                    "metadata": {}
+                }
+
+                a_file.write(json.dumps(ans_item) + '\n')
+                a_file.flush()
+    elif vlm == 'bing':
+        from approach.vlm.bing.bing import context_conversation
+        with open(vlm_question_path, 'r') as q_file, open(vlm_answer_path, 'a') as a_file:  # Open answer file in append mode
+            for line in tqdm(q_file):
+                ans_item = {}
+
+                line_data = json.loads(line)
+                image_path = os.path.join(images_dir, line_data['image'])
+                image_question = line_data['text']
+
+                bing_res = context_conversation(image_question, image_path)
+                print(bing_res)
+                time.sleep(15)
+
+                ans_item = {
+                    "question_id": line_data['question_id'],
+                    "prompt": image_question,
+                    "text": bing_res,
+                    "answer_id": generate_answer_id(),
+                    "model_id": vlm,
+                    "metadata": {}
+                }
+
+                a_file.write(json.dumps(ans_item) + '\n')
+    elif vlm == 'gemini':
+        import pathlib
+        import textwrap
+
+        import google.generativeai as genai
+
+        GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "[GOOGLE_API_KEY]")
+        genai.configure(api_key=GOOGLE_API_KEY)
+
+        safety_settings = [
+            {
+                "category": "HARM_CATEGORY_DANGEROUS",
+                "threshold": "BLOCK_NONE",
+            },
+            {
+                "category": "HARM_CATEGORY_HARASSMENT",
+                "threshold": "BLOCK_NONE",
+            },
+            {
+                "category": "HARM_CATEGORY_HATE_SPEECH",
+                "threshold": "BLOCK_NONE",
+            },
+            {
+                "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+                "threshold": "BLOCK_NONE",
+            },
+            {
+                "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+                "threshold": "BLOCK_NONE",
+            },
+        ]
+
+        # before fse
+        # model = genai.GenerativeModel('gemini-pro-vision')
+        # fse
+        model = genai.GenerativeModel('gemini-1.5-pro')
+
+        with open(vlm_question_path, 'r') as q_file, open(gemini_vlm_answer_path, 'w') as a_file:  # Open answer file in append mode
+            qfile_lines = q_file.readlines()[0:2]
+            for line in tqdm(qfile_lines):
+
+                ans_item = {}
+
+                line_data = json.loads(line)
+                image_path = os.path.join(images_dir, line_data['image'])
+                image_question = line_data['text']
+
+                gemini_response = model.generate_content([image_question, PIL.Image.open(image_path)], safety_settings=safety_settings)
+                gemini_response.resolve()
+                try:
+                    gemini_response = gemini_response.text
+                    ans_item = {
+                        "question_id": line_data['question_id'],
+                        "prompt": image_question,
+                        "text": gemini_response,
+                        "answer_id": generate_answer_id(),
+                        "model_id": vlm,
+                        "metadata": {}
+                    }
+
+                    a_file.write(json.dumps(ans_item) + '\n')
+                    a_file.flush()
+                except Exception as e:
+                    print(f"Error for image {image_path}. Error: {e}")
+                # print(gemini_response)
+                time.sleep(30)
+
+
+
+    elif vlm.endswith('_pass'):
+        print(f'Passing VLM {vlm} ...')
+    else:
+        raise Exception('Unrecognized VLM!')
+
+    logging.info(f'STEP #1 VLM {vlm} analysis completed at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    logging.info(f'STEP #1 VLM {vlm} analysis total time taken: {time.time() - vlm_start_time} seconds')
+
+
+
+
+
+    # Get label candidates
+    # if llm == 'gemini-3-flash-preview-nothinking':
+    #     from approach.llm.gpt_polling import infer_object_candidates
+    # elif llm == 'llama2':
+    #     from approach.llm.llama import infer_object_candidates
+
+
+    # STEP #3
+    if llm == 'gemini-3-flash-preview-nothinking' or llm == 'gemini-3-flash-preview-nothinking_abl':
+        from approach.llm.gpt_polling import infer_objects
+
+        if llm == 'gemini-3-flash-preview-nothinking_abl':
+            gpt35_ablation = True
+        elif llm == 'gemini-3-flash-preview-nothinking':
+            gpt35_ablation = False
+        infer_objects(vlm_question_path, vlm_answer_path, interactable_object_path, gpt35_ablation)
+    elif llm == 'llama2':
+        from approach.llm.llama import infer_objects
+        exit(0)
+    elif llm.endswith('_pass'):
+        print(f'Passing LLM {llm} ...')
+    else:
+        raise Exception('Unrecognized LLM!')
+
+
+
+
+
+
+    # STEP #2
+    # Open-vocabulary object detection
+    ovod_start_time = time.time()
+    logging.info(f'STEP #2 OVOD {ovod} analysis started at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    vlm_questions = {}
+    with open(complete_vlm_question_path, 'r') as q_file:
+        for line in q_file:
+            line_data = json.loads(line)
+            vlm_questions[line_data['question_id']] = line_data
+            # print(line_data['question_id'])
+
+    if ovod == 'grounding_dino':
+        from approach.ovod.GroundingDINO.demo.inference_on_a_image import process_grounding_dino
+
+        original_path = os.path.dirname(__file__)
+        os.chdir(grounding_dino_path)
+
+        # with open(llm_candidate_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        with open(interactable_object_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+            llm_candidate = json.load(llm_candidate_file)
+            all_oovd_res = {}
+            for image_index in llm_candidate.keys():
+                print(image_index)
+                 # TODO:
+                if (image_index != '1160'):
+                    ovod_image_path = os.path.join(images_dir, vlm_questions[int(image_index)]["image"])
+                    ovod_candidates = llm_candidate[image_index]['interactable_objects']
+                    # grounding_dino_command = generate_grounding_dino_command(ovod_image_path, ovod_output_dir, ovod_candidates)
+                    # print(grounding_dino_command)
+                    # os.system(grounding_dino_command)
+
+                    gdino_res = process_grounding_dino(
+                        config_file='groundingdino/config/GroundingDINO_SwinT_OGC.py',
+                        checkpoint_path='weights/groundingdino_swint_ogc.pth',
+                        image_path=ovod_image_path,
+                        ovod_candidates=ovod_candidates,
+                        output_dir=ovod_output_dir,
+                        box_threshold=0.3,
+                        text_threshold=0.25,
+                        token_spans=None,
+                        cpu_only=False
+                    )
+                    # print(gdino_res)
+
+                    object_oovd_item = copy.deepcopy(llm_candidate[image_index])
+                    object_oovd_item['oovd_result'] = gdino_res
+                    print(object_oovd_item)
+                    all_oovd_res[image_index] = object_oovd_item
+
+            json.dump(all_oovd_res, oovd_file, indent=4)
+
+
+        os.chdir(original_path)
+
+    elif ovod == 'ape_d' or ovod == 'ape_d_abl':
+        from approach.ovod.APE.demo.ape_inference import run_ape_model_inference
+
+        original_path = os.path.dirname(__file__)
+        os.chdir(active_ape_path)
+
+        all_ape_res = []
+        reflection_traces = []
+        # General
+        # with open(vlm_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+        # GPT-4 eval
+        with open(gemini31_results_answer_path, 'r') as llm_candidate_file:
+        # Gemini abl i eval
+        # with open(interactability_abl_results_answer_path, 'r') as llm_candidate_file, open(oovd_object_bbox_path, 'w') as oovd_file:
+
+            # llm_candidate = json.load(llm_candidate_file)
+            all_oovd_res = {}
+
+            candidate_lines = select_jsonl_lines(
+                llm_candidate_file.readlines(),
+                start_index=start_index,
+                end_index=end_index,
+                shard_index=shard_index,
+                num_shards=num_shards,
+            )
+
+            for line in tqdm(candidate_lines):
+                ans_item = {}
+
+                line_data = json.loads(line)
+
+            # for image_index in llm_candidate.keys():
+                # print(image_index)
+                 # TODO:
+                # if (image_index != '1160'):
+                # image_index = line_data['question_id']
+
+                image_name = vlm_questions[line_data['question_id']]["image"]
+                ovod_image_path = os.path.join(images_dir, image_name)
+
+                # General
+                # try:
+                #     if line_data['text'].startswith(" ```json"):
+                #         ovod_candidates = json.loads(line_data['text'][8:-4])['objects']
+                #     elif line_data['text'].startswith(' {\"objects\"'):
+                #         ovod_candidates = json.loads(line_data['text'])['objects']
+                #     else:
+                #         print(f"Error for decoding IVO json for {image_name}.")
+                # except Exception as e:
+                #     print(f"Error for decoding IVO json for image {image_name}. Error: {e}")
+                #     continue
+
+
+                # GPT-4v
+                if line_data['text']:
+                    ovod_candidates = line_data['text']['objects']
+                else:
+                    continue
+                all_res = []
+                for ocd in ovod_candidates.keys():
+                    referring_expr_str = ovod_candidates[ocd]
+
+                    translator = str.maketrans('', '', string.punctuation)
+                    if type(referring_expr_str) is str:
+                        referring_expr_str = referring_expr_str.translate(translator)
+                    else:
+                        # dict
+                        referring_expr_str = ' '.join(referring_expr_str.values())
+                        referring_expr_str = referring_expr_str.translate(translator)
+                    referring_expr_str = f'{ocd}: {referring_expr_str}'
+                    # print(image_name, referring_expr_str)
+                    all_res.append(referring_expr_str)
+
+                if ovod == 'ape_d':
+                    ape_threshold = 0.1
+                elif ovod == 'ape_d_abl':
+                    ape_threshold = 0.1
+
+                ape_res = []
+                try:
+                    os.makedirs(f'./realfse_{vlm}_{ovod}_gpu0123', exist_ok=True)
+                    ape_res = run_ape_model_inference(
+                        config_file='configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py',
+                        input_path=ovod_image_path,
+                        # GPU 0
+                        output_path=f'./realfse_{vlm}_{ovod}_gpu0123',
+                        confidence_threshold=ape_threshold,
+                        text_prompt=', '.join(all_res),
+                        with_box=True,
+                        with_mask=False,
+                        with_sseg=False,
+                        opts=[
+                                f"train.init_checkpoint='{active_ape_checkpoint}'",
+                                "model.model_language.cache_dir=''",
+                                "model.model_vision.select_box_nums_for_evaluation=500",
+                                "model.model_vision.text_feature_bank_reset=True",
+                                "model.model_vision.backbone.net.xattn=False",
+                                "model.model_vision.transformer.encoder.pytorch_attn=True",
+                                "model.model_vision.transformer.decoder.pytorch_attn=True"
+                        ]
+                    )
+                except Exception as e:
+                    print(f"Error for image {image_name}. Error: {e}")
+
+                def reflection_detector(candidates, previous):
+                    if not candidates:
+                        return previous
+                    try:
+                        refined = run_ape_model_inference(
+                            config_file='configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py',
+                            input_path=ovod_image_path,
+                            output_path=f'./realfse_{vlm}_{ovod}_gpu0123',
+                            confidence_threshold=ape_threshold,
+                            text_prompt=', '.join(candidates),
+                            with_box=True,
+                            with_mask=False,
+                            with_sseg=False,
+                            opts=[
+                                    f"train.init_checkpoint='{active_ape_checkpoint}'",
+                                    "model.model_language.cache_dir=''",
+                                    "model.model_vision.select_box_nums_for_evaluation=500",
+                                    "model.model_vision.text_feature_bank_reset=True",
+                                    "model.model_vision.backbone.net.xattn=False",
+                                    "model.model_vision.transformer.encoder.pytorch_attn=True",
+                                    "model.model_vision.transformer.decoder.pytorch_attn=True"
+                            ]
+                        )
+                        return previous + refined
+                    except Exception as e:
+                        print(f"Error during reflection redetection for {image_name}. Error: {e}")
+                        return previous
+
+                reflection_result = run_optional_reflection(
+                    ovod_image_path,
+                    ape_res,
+                    detector=reflection_detector,
+                    enabled=enable_reflection,
+                    model_profile=reflection_profile,
+                    max_iterations=max_reflection_iterations,
+                )
+                if reflection_result is not None:
+                    ape_res = reflection_result["detections"]
+                    reflection_traces.append(
+                        {
+                            "image": image_name,
+                            "image_id": extract_image_id(image_name),
+                            "trace": reflection_result["trace"],
+                            "max_iterations_reached": reflection_result["max_iterations_reached"],
+                        }
+                    )
+
+                all_ape_res.extend(enrich_ape_results(ape_res, image_name, extract_image_id))
+                # object_oovd_item = copy.deepcopy(llm_candidate[image_index])
+                # object_oovd_item['ape_result'] = all_ape_res
+                # # print(object_oovd_item)
+                # all_oovd_res[image_index] = object_oovd_item
+
+            write_json_atomic(oovd_object_bbox_path, all_ape_res)
+            if enable_reflection:
+                write_json_atomic(f"{oovd_object_bbox_path}.reflection.json", reflection_traces)
+
+
+        os.chdir(original_path)
+
+    elif ovod.endswith('_pass'):
+        print(f'Passing OVOD {ovod} ...')
+    else:
+        raise Exception('Unrecognized OVOD!')
+
+    logging.info(f'STEP #2 OVOD {ovod} analysis completed at %s', time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
+    logging.info(f'STEP #2 OVOD {ovod} analysis total time taken: {time.time() - ovod_start_time} seconds')
+# icse & fse
+def extract_image_id(image_name):
+    # Split the image name into parts and form the image_id accordingly
+    parts = image_name.split('_')
+    print(image_name)
+    base, extension = parts[1].split('.')
+    return int(parts[0] + base.zfill(3))
+# icse_rebuttal
+# def extract_image_id(image_name):
+#     # Split the image name into parts by underscore
+#     parts = image_name.split('_')
+#     # Extracting the first part as the base and the numeric portion of the third part before the file extension
+#     base = parts[0]  # This will give '625470'
+#     numeric_part = parts[2].split('.')[0]  # This will give 'b3'
+#     # Removing non-numeric characters from 'b3'
+#     numeric_part = ''.join(filter(str.isdigit, numeric_part))
+#     # Zfill is used to ensure the numeric part has at least 3 digits, then combining with base
+#     print(int(base + numeric_part.zfill(3)))
+#     return int(base + numeric_part.zfill(3))
+
+
+def main():
+    vlms = ['llava7b', 'bing', 'gpt4v']
+    llms = ['llama2', 'gemini-3-flash-preview-nothinking']
+    ovods = ['grounding_dino', 'glip', 'ape_d']
+
+
+# python method.py -v gpt4v -l gemini-3-flash-preview-nothinking -o ape_d
+# python method.py -v gpt4v -l gemini-3-flash-preview-nothinking -o ape_d_abl
+# python method.py -v claude35sonnet -l gemini-3-flash-preview-nothinking -o ape_d
+# python method.py -v gemini -l gemini-3-flash-preview-nothinking -o ape_d
+if __name__=='__main__':
+    # main()
+
+    parser = argparse.ArgumentParser("Method", add_help=True)
+    parser.add_argument("--vlm", "-v", type=str, required=True, help="vlm")
+    parser.add_argument("--llm", "-l", type=str, required=True, help="llm")
+    parser.add_argument("--ovod", "-o", type=str, required=True, help="ovod")
+    parser.add_argument("--start-index", type=int, default=None, help="first JSONL row to process")
+    parser.add_argument("--end-index", type=int, default=None, help="exclusive JSONL row end")
+    parser.add_argument("--shard-index", type=int, default=None, help="zero-based shard index")
+    parser.add_argument("--num-shards", type=int, default=None, help="total number of shards")
+    parser.add_argument("--enable-reflection", action="store_true", help="run PII.5/PII.6 advisor reflection loop")
+    parser.add_argument("--reflection-profile", default="default", help="model profile for the reflection advisor")
+    parser.add_argument("--max-reflection-iterations", type=int, default=10, help="max advisor reflection rounds")
+    parser.add_argument("--questions", dest="questions_path", help="question manifest JSONL")
+    parser.add_argument("--candidates", dest="candidates_path", help="candidate JSON/JSONL for the selected detector")
+    parser.add_argument("--images-dir", dest="images_path", help="directory containing XR screenshots")
+    parser.add_argument("--output", dest="output_path", help="prediction JSON path; selection suffixes are added automatically")
+    parser.add_argument("--ape-root", help="APE repository directory")
+    parser.add_argument("--ape-checkpoint", help="APE checkpoint path")
+    parser.add_argument("--log-file", help="optional log file; defaults to stderr")
+    args = parser.parse_args()
+
+    gpu = '1-3'
+    stage = '1'
+    idx = '0001'
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+        datefmt='%m/%d/%Y %H:%M:%S',
+        level=logging.INFO,
+        filename=args.log_file,
+        )
+
+    method(
+        args.vlm,
+        args.llm,
+        args.ovod,
+        start_index=args.start_index,
+        end_index=args.end_index,
+        shard_index=args.shard_index,
+        num_shards=args.num_shards,
+        enable_reflection=args.enable_reflection,
+        reflection_profile=args.reflection_profile,
+        max_reflection_iterations=args.max_reflection_iterations,
+        questions_path=args.questions_path,
+        candidates_path=args.candidates_path,
+        images_path=args.images_path,
+        output_path=args.output_path,
+        ape_root=args.ape_root,
+        ape_checkpoint=args.ape_checkpoint,
+    )
+
+# CUDA_VISIBLE_DEVICES=0 python method.py -v gpt4v -l gemini-3-flash-preview-nothinking -o ape_d > ../log/realfse/240910_gpt4v_2_g0_0001.txt
+# CUDA_VISIBLE_DEVICES=3 python method.py -v gpt4v -l gemini-3-flash-preview-nothinking -o ape_d_abl > ../log/realfse/240910_gpt4v_ape_d_abl_2_g3_0001.txt
+# CUDA_VISIBLE_DEVICES=3 python method.py -v gpt4v_abl -l gemini-3-flash-preview-nothinking -o ape_d > ../log/realfse/240912_gpt4v_abl_1_g1-3_0001.txt

approach/ovod/APE/demo/pre-requirements.txt

@@ -0,0 +1,4 @@
+--index-url https://download.pytorch.org/whl/cu118
+torch==2.0.1
+torchvision==0.15.2
+torchaudio==2.0.2

approach/ovod/APE/demo/predictor_lazy.py

@@ -0,0 +1,429 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+import atexit
+import bisect
+import gc
+import json
+import multiprocessing as mp
+import time
+from collections import deque
+
+import cv2
+import numpy as np
+import torch
+
+from ape.engine.defaults import DefaultPredictor
+from detectron2.data import MetadataCatalog
+from detectron2.utils.video_visualizer import VideoVisualizer
+from detectron2.utils.visualizer import ColorMode, Visualizer
+
+
+def filter_instances(instances, metadata):
+    # return instances
+
+    keep = []
+    keep_classes = []
+
+    sorted_idxs = np.argsort(-instances.scores)
+    instances = instances[sorted_idxs]
+
+    for i in range(len(instances)):
+        instance = instances[i]
+        pred_class = instance.pred_classes
+        if pred_class >= len(metadata.thing_classes):
+            continue
+
+        keep.append(i)
+        keep_classes.append(pred_class)
+    return instances[keep]
+
+
+def cuda_grabcut(img, masks, iter=5, gamma=50, iou_threshold=0.75):
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    try:
+        import grabcut
+    except Exception as e:
+        print("*" * 60)
+        print("fail to import grabCut: ", e)
+        print("*" * 60)
+        return masks
+    GC = grabcut.GrabCut(iter)
+
+    img = cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)
+
+    tic_0 = time.time()
+    for i in range(len(masks)):
+        mask = masks[i]
+        if mask.sum() > 10 * 10:
+            pass
+        else:
+            continue
+
+        # ----------------------------------------------------------------
+        fourmap = np.empty_like(mask, dtype=np.uint8)
+        fourmap[:, :] = 64
+        fourmap[mask == 0] = 64
+        fourmap[mask == 1] = 128
+
+        # Compute segmentation
+        tic = time.time()
+        seg = GC.estimateSegmentationFromFourmap(img, fourmap, gamma)
+        toc = time.time()
+        print("Time elapsed in GrabCut segmentation: " + str(toc - tic))
+        # ----------------------------------------------------------------
+
+        seg = torch.tensor(seg, dtype=torch.bool)
+        iou = (mask & seg).sum() / (mask | seg).sum()
+        if iou > iou_threshold:
+            masks[i] = seg
+
+        if toc - tic_0 > 10:
+            break
+
+    return masks
+
+
+def opencv_grabcut(img, masks, iter=5):
+
+    for i in range(len(masks)):
+        mask = masks[i]
+
+        # ----------------------------------------------------------------
+        fourmap = np.empty_like(mask, dtype=np.uint8)
+        fourmap[:, :] = cv2.GC_PR_BGD
+        # fourmap[mask == 0] = cv2.GC_BGD
+        fourmap[mask == 0] = cv2.GC_PR_BGD
+        fourmap[mask == 1] = cv2.GC_PR_FGD
+        # fourmap[mask == 1] = cv2.GC_FGD
+
+        # Create GrabCut algo
+        bgd_model = np.zeros((1, 65), np.float64)
+        fgd_model = np.zeros((1, 65), np.float64)
+        seg = np.zeros_like(fourmap, dtype=np.uint8)
+
+        # Compute segmentation
+        tic = time.time()
+        seg, bgd_model, fgd_model = cv2.grabCut(
+            img, fourmap, None, bgd_model, fgd_model, iter, cv2.GC_INIT_WITH_MASK
+        )
+        toc = time.time()
+        print("Time elapsed in GrabCut segmentation: " + str(toc - tic))
+
+        seg = np.where((seg == 2) | (seg == 0), 0, 1).astype("bool")
+
+        # ----------------------------------------------------------------
+
+        seg = torch.tensor(seg, dtype=torch.bool)
+        iou = (mask & seg).sum() / (mask | seg).sum()
+        if iou > 0.75:
+            masks[i] = seg
+
+        if i > 10:
+            break
+
+    return masks
+
+
+class VisualizationDemo(object):
+    def __init__(self, cfg, instance_mode=ColorMode.IMAGE, parallel=False, args=None):
+        """
+        Args:
+            cfg (CfgNode):
+            instance_mode (ColorMode):
+            parallel (bool): whether to run the model in different processes from visualization.
+                Useful since the visualization logic can be slow.
+        """
+        self.metadata = MetadataCatalog.get(
+            "__unused_" + "_".join([d for d in cfg.dataloader.train.dataset.names])
+        )
+        self.metadata.thing_classes = [
+            c
+            for d in cfg.dataloader.train.dataset.names
+            for c in MetadataCatalog.get(d).get("thing_classes", default=[])
+            + MetadataCatalog.get(d).get("stuff_classes", default=["thing"])[1:]
+        ]
+        self.metadata.stuff_classes = [
+            c
+            for d in cfg.dataloader.train.dataset.names
+            for c in MetadataCatalog.get(d).get("thing_classes", default=[])
+            + MetadataCatalog.get(d).get("stuff_classes", default=["thing"])[1:]
+        ]
+
+        # self.metadata = MetadataCatalog.get(
+        #     "__unused_ape_" + "_".join([d for d in cfg.dataloader.train.dataset.names])
+        # )
+        # self.metadata.thing_classes = [
+        #     c
+        #     for d in ["coco_2017_train_panoptic_separated"]
+        #     for c in MetadataCatalog.get(d).get("thing_classes", default=[])
+        #     + MetadataCatalog.get(d).get("stuff_classes", default=["thing"])[1:]
+        # ]
+        # self.metadata.stuff_classes = [
+        #     c
+        #     for d in ["coco_2017_train_panoptic_separated"]
+        #     for c in MetadataCatalog.get(d).get("thing_classes", default=[])
+        #     + MetadataCatalog.get(d).get("stuff_classes", default=["thing"])[1:]
+        # ]
+
+        self.cpu_device = torch.device("cpu")
+        self.instance_mode = instance_mode
+
+        self.parallel = parallel
+        if parallel:
+            num_gpu = torch.cuda.device_count()
+            self.predictor = AsyncPredictor(cfg, num_gpus=num_gpu)
+        else:
+            self.predictor = DefaultPredictor(cfg)
+
+        print(args)
+
+    def run_on_image(
+        self,
+        image,
+        text_prompt=None,
+        mask_prompt=None,
+        with_box=True,
+        with_mask=True,
+        with_sseg=True,
+    ):
+        """
+        Args:
+            image (np.ndarray): an image of shape (H, W, C) (in BGR order).
+                This is the format used by OpenCV.
+
+        Returns:
+            predictions (dict): the output of the model.
+            vis_output (VisImage): the visualized image output.
+        """
+        if text_prompt:
+            text_list = [x.strip() for x in text_prompt.split(",")]
+            text_list = [x for x in text_list if len(x) > 0]
+            metadata = MetadataCatalog.get("__unused_ape_" + text_prompt)
+            metadata.thing_classes = text_list
+            metadata.stuff_classes = text_list
+        else:
+            metadata = self.metadata
+
+        vis_output = None
+        predictions = self.predictor(image, text_prompt, mask_prompt)
+
+        if "instances" in predictions:
+            predictions["instances"] = filter_instances(
+                predictions["instances"].to(self.cpu_device), metadata
+            )
+
+        # Convert image from OpenCV BGR format to Matplotlib RGB format.
+        image = image[:, :, ::-1]
+        visualizer = Visualizer(image, metadata, instance_mode=self.instance_mode)
+        vis_outputs = []
+        if "panoptic_seg" in predictions and with_mask and with_sseg:
+            panoptic_seg, segments_info = predictions["panoptic_seg"]
+            vis_output = visualizer.draw_panoptic_seg_predictions(
+                panoptic_seg.to(self.cpu_device), segments_info
+            )
+        else:
+            if "sem_seg" in predictions and with_sseg:
+                # vis_output = visualizer.draw_sem_seg(
+                #     predictions["sem_seg"].argmax(dim=0).to(self.cpu_device)
+                # )
+
+                sem_seg = predictions["sem_seg"].to(self.cpu_device)
+                # sem_seg = opencv_grabcut(image, sem_seg, iter=10)
+                # sem_seg = cuda_grabcut(image, sem_seg > 0.5, iter=5, gamma=10, iou_threshold=0.1)
+                sem_seg = torch.cat((sem_seg, torch.ones_like(sem_seg[0:1, ...]) * 0.1), dim=0)
+                sem_seg = sem_seg.argmax(dim=0)
+                vis_output = visualizer.draw_sem_seg(sem_seg)
+            if "instances" in predictions and (with_box or with_mask):
+                instances = predictions["instances"].to(self.cpu_device)
+
+                if not with_box:
+                    instances.remove("pred_boxes")
+                if not with_mask:
+                    instances.remove("pred_masks")
+
+                if with_mask and False:
+                    # instances.pred_masks = opencv_grabcut(image, instances.pred_masks, iter=10)
+                    instances.pred_masks = cuda_grabcut(
+                        image, instances.pred_masks, iter=5, gamma=10, iou_threshold=0.75
+                    )
+
+                vis_output = visualizer.draw_instance_predictions(predictions=instances)
+
+                # for i in range(len(instances)):
+                #     visualizer = Visualizer(image, metadata, instance_mode=self.instance_mode)
+                #     vis_outputs.append(visualizer.draw_instance_predictions(predictions=instances[i]))
+
+            elif "proposals" in predictions:
+                visualizer = Visualizer(image, None, instance_mode=self.instance_mode)
+                instances = predictions["proposals"].to(self.cpu_device)
+                instances.pred_boxes = instances.proposal_boxes
+                instances.scores = instances.objectness_logits
+                vis_output = visualizer.draw_instance_predictions(predictions=instances)
+
+        return predictions, vis_output, vis_outputs, metadata
+
+    def _frame_from_video(self, video):
+        while video.isOpened():
+            success, frame = video.read()
+            if success:
+                yield frame
+            else:
+                break
+
+    def run_on_video(self, video):
+        """
+        Visualizes predictions on frames of the input video.
+
+        Args:
+            video (cv2.VideoCapture): a :class:`VideoCapture` object, whose source can be
+                either a webcam or a video file.
+
+        Yields:
+            ndarray: BGR visualizations of each video frame.
+        """
+        video_visualizer = VideoVisualizer(self.metadata, self.instance_mode)
+
+        def process_predictions(frame, predictions):
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            if "panoptic_seg" in predictions and False:
+                panoptic_seg, segments_info = predictions["panoptic_seg"]
+                vis_frame = video_visualizer.draw_panoptic_seg_predictions(
+                    frame, panoptic_seg.to(self.cpu_device), segments_info
+                )
+            elif "instances" in predictions and False:
+                predictions = predictions["instances"].to(self.cpu_device)
+                vis_frame = video_visualizer.draw_instance_predictions(frame, predictions)
+            elif "sem_seg" in predictions and False:
+                vis_frame = video_visualizer.draw_sem_seg(
+                    frame, predictions["sem_seg"].argmax(dim=0).to(self.cpu_device)
+                )
+
+            if "sem_seg" in predictions:
+                vis_frame = video_visualizer.draw_sem_seg(
+                    frame, predictions["sem_seg"].argmax(dim=0).to(self.cpu_device)
+                )
+                frame = vis_frame.get_image()
+
+            if "instances" in predictions:
+                predictions = predictions["instances"].to(self.cpu_device)
+                predictions = filter_instances(predictions, self.metadata)
+                vis_frame = video_visualizer.draw_instance_predictions(frame, predictions)
+
+            # Converts Matplotlib RGB format to OpenCV BGR format
+            vis_frame = cv2.cvtColor(vis_frame.get_image(), cv2.COLOR_RGB2BGR)
+            return vis_frame, predictions
+
+        frame_gen = self._frame_from_video(video)
+        if self.parallel:
+            buffer_size = self.predictor.default_buffer_size
+
+            frame_data = deque()
+
+            for cnt, frame in enumerate(frame_gen):
+                frame_data.append(frame)
+                self.predictor.put(frame)
+
+                if cnt >= buffer_size:
+                    frame = frame_data.popleft()
+                    predictions = self.predictor.get()
+                    yield process_predictions(frame, predictions)
+
+            while len(frame_data):
+                frame = frame_data.popleft()
+                predictions = self.predictor.get()
+                yield process_predictions(frame, predictions)
+        else:
+            for frame in frame_gen:
+                yield process_predictions(frame, self.predictor(frame))
+
+
+class AsyncPredictor:
+    """
+    A predictor that runs the model asynchronously, possibly on >1 GPUs.
+    Because rendering the visualization takes considerably amount of time,
+    this helps improve throughput a little bit when rendering videos.
+    """
+
+    class _StopToken:
+        pass
+
+    class _PredictWorker(mp.Process):
+        def __init__(self, cfg, task_queue, result_queue):
+            self.cfg = cfg
+            self.task_queue = task_queue
+            self.result_queue = result_queue
+            super().__init__()
+
+        def run(self):
+            predictor = DefaultPredictor(self.cfg)
+
+            while True:
+                task = self.task_queue.get()
+                if isinstance(task, AsyncPredictor._StopToken):
+                    break
+                idx, data = task
+                result = predictor(data)
+                self.result_queue.put((idx, result))
+
+    def __init__(self, cfg, num_gpus: int = 1):
+        """
+        Args:
+            cfg (CfgNode):
+            num_gpus (int): if 0, will run on CPU
+        """
+        num_workers = max(num_gpus, 1)
+        self.task_queue = mp.Queue(maxsize=num_workers * 3)
+        self.result_queue = mp.Queue(maxsize=num_workers * 3)
+        self.procs = []
+        for gpuid in range(max(num_gpus, 1)):
+            cfg = cfg.clone()
+            cfg.defrost()
+            cfg.MODEL.DEVICE = "cuda:{}".format(gpuid) if num_gpus > 0 else "cpu"
+            self.procs.append(
+                AsyncPredictor._PredictWorker(cfg, self.task_queue, self.result_queue)
+            )
+
+        self.put_idx = 0
+        self.get_idx = 0
+        self.result_rank = []
+        self.result_data = []
+
+        for p in self.procs:
+            p.start()
+        atexit.register(self.shutdown)
+
+    def put(self, image):
+        self.put_idx += 1
+        self.task_queue.put((self.put_idx, image))
+
+    def get(self):
+        self.get_idx += 1  # the index needed for this request
+        if len(self.result_rank) and self.result_rank[0] == self.get_idx:
+            res = self.result_data[0]
+            del self.result_data[0], self.result_rank[0]
+            return res
+
+        while True:
+            # make sure the results are returned in the correct order
+            idx, res = self.result_queue.get()
+            if idx == self.get_idx:
+                return res
+            insert = bisect.bisect(self.result_rank, idx)
+            self.result_rank.insert(insert, idx)
+            self.result_data.insert(insert, res)
+
+    def __len__(self):
+        return self.put_idx - self.get_idx
+
+    def __call__(self, image):
+        self.put(image)
+        return self.get()
+
+    def shutdown(self):
+        for _ in self.procs:
+            self.task_queue.put(AsyncPredictor._StopToken())
+
+    @property
+    def default_buffer_size(self):
+        return len(self.procs) * 5

approach/ovod/APE/demo/requirements.txt

@@ -0,0 +1,11 @@
+transformers
+cython
+opencv-python
+scipy
+einops
+lvis
+fairscale
+git+https://github.com/facebookresearch/detectron2@017abbf
+git+https://github.com/IDEA-Research/detrex@776058e
+git+https://github.com/openai/CLIP.git@d50d76d
+git+https://github.com/shenyunhang/ape

approach/ovod/GroundingDINO/.gitignore

@@ -0,0 +1,146 @@
+# IDE
+.idea/
+.vscode/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# vscode
+.vscode/
+output/
+outputs/
+subs/
+logs/
+
+grounding/config/configs
+grounding/version.py
+
+vis/
+tmp/

approach/ovod/GroundingDINO/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 - present, IDEA Research.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

approach/ovod/GroundingDINO/README.md

@@ -0,0 +1,367 @@
+<div align="center">
+  <img src="./.asset/grounding_dino_logo.png" width="30%">
+</div>
+
+# :sauropod: Grounding DINO 
+
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/grounding-dino-marrying-dino-with-grounded/zero-shot-object-detection-on-mscoco)](https://paperswithcode.com/sota/zero-shot-object-detection-on-mscoco?p=grounding-dino-marrying-dino-with-grounded) [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/grounding-dino-marrying-dino-with-grounded/zero-shot-object-detection-on-odinw)](https://paperswithcode.com/sota/zero-shot-object-detection-on-odinw?p=grounding-dino-marrying-dino-with-grounded) \
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/grounding-dino-marrying-dino-with-grounded/object-detection-on-coco-minival)](https://paperswithcode.com/sota/object-detection-on-coco-minival?p=grounding-dino-marrying-dino-with-grounded) [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/grounding-dino-marrying-dino-with-grounded/object-detection-on-coco)](https://paperswithcode.com/sota/object-detection-on-coco?p=grounding-dino-marrying-dino-with-grounded)
+
+
+**[IDEA-CVR, IDEA-Research](https://github.com/IDEA-Research)** 
+
+[Shilong Liu](http://www.lsl.zone/), [Zhaoyang Zeng](https://scholar.google.com/citations?user=U_cvvUwAAAAJ&hl=zh-CN&oi=ao), [Tianhe Ren](https://rentainhe.github.io/), [Feng Li](https://scholar.google.com/citations?user=ybRe9GcAAAAJ&hl=zh-CN), [Hao Zhang](https://scholar.google.com/citations?user=B8hPxMQAAAAJ&hl=zh-CN), [Jie Yang](https://github.com/yangjie-cv), [Chunyuan Li](https://scholar.google.com/citations?user=Zd7WmXUAAAAJ&hl=zh-CN&oi=ao), [Jianwei Yang](https://jwyang.github.io/), [Hang Su](https://scholar.google.com/citations?hl=en&user=dxN1_X0AAAAJ&view_op=list_works&sortby=pubdate), [Jun Zhu](https://scholar.google.com/citations?hl=en&user=axsP38wAAAAJ), [Lei Zhang](https://www.leizhang.org/)<sup>:email:</sup>.
+
+
+[[`Paper`](https://arxiv.org/abs/2303.05499)] [[`Demo`](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo)] [[`BibTex`](#black_nib-citation)]
+
+
+PyTorch implementation and pretrained models for Grounding DINO. For details, see the paper **[Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection](https://arxiv.org/abs/2303.05499)**.
+
+## :sun_with_face: Helpful Tutorial
+
+- :grapes: [[Read our arXiv Paper](https://arxiv.org/abs/2303.05499)]
+- :apple:  [[Watch our simple introduction video on YouTube](https://youtu.be/wxWDt5UiwY8)]
+- :blossom:   &nbsp;[[Try the Colab Demo](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/zero-shot-object-detection-with-grounding-dino.ipynb)]
+- :sunflower: [[Try our Official Huggingface Demo](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo)]
+- :maple_leaf: [[Watch the Step by Step Tutorial about GroundingDINO by Roboflow AI](https://youtu.be/cMa77r3YrDk)]
+- :mushroom: [[GroundingDINO: Automated Dataset Annotation and Evaluation by Roboflow AI](https://youtu.be/C4NqaRBz_Kw)]
+- :hibiscus: [[Accelerate Image Annotation with SAM and GroundingDINO by Roboflow AI](https://youtu.be/oEQYStnF2l8)]
+- :white_flower: [[Autodistill: Train YOLOv8 with ZERO Annotations based on Grounding-DINO and Grounded-SAM by Roboflow AI](https://github.com/autodistill/autodistill)]
+
+<!-- Grounding DINO Methods | 
+[![arXiv](https://img.shields.io/badge/arXiv-2303.05499-b31b1b.svg)](https://arxiv.org/abs/2303.05499) 
+[![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://youtu.be/wxWDt5UiwY8) -->
+
+<!-- Grounding DINO Demos |
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/zero-shot-object-detection-with-grounding-dino.ipynb) -->
+<!-- [![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://youtu.be/cMa77r3YrDk)
+[![HuggingFace space](https://img.shields.io/badge/🤗-HuggingFace%20Space-cyan.svg)](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo)
+[![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://youtu.be/oEQYStnF2l8)
+[![YouTube](https://badges.aleen42.com/src/youtube.svg)](https://youtu.be/C4NqaRBz_Kw) -->
+
+## :sparkles: Highlight Projects
+
+- [Semantic-SAM: a universal image segmentation model to enable segment and recognize anything at any desired granularity.](https://github.com/UX-Decoder/Semantic-SAM), 
+- [DetGPT: Detect What You Need via Reasoning](https://github.com/OptimalScale/DetGPT)
+- [Grounded-SAM: Marrying Grounding DINO with Segment Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything)
+- [Grounding DINO with Stable Diffusion](demo/image_editing_with_groundingdino_stablediffusion.ipynb)
+- [Grounding DINO with GLIGEN for Controllable Image Editing](demo/image_editing_with_groundingdino_gligen.ipynb)
+- [OpenSeeD: A Simple and Strong Openset Segmentation Model](https://github.com/IDEA-Research/OpenSeeD)
+- [SEEM: Segment Everything Everywhere All at Once](https://github.com/UX-Decoder/Segment-Everything-Everywhere-All-At-Once)
+- [X-GPT: Conversational Visual Agent supported by X-Decoder](https://github.com/microsoft/X-Decoder/tree/xgpt)
+- [GLIGEN: Open-Set Grounded Text-to-Image Generation](https://github.com/gligen/GLIGEN)
+- [LLaVA: Large Language and Vision Assistant](https://github.com/haotian-liu/LLaVA)
+
+<!-- Extensions | [Grounding DINO with Segment Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything); [Grounding DINO with Stable Diffusion](demo/image_editing_with_groundingdino_stablediffusion.ipynb); [Grounding DINO with GLIGEN](demo/image_editing_with_groundingdino_gligen.ipynb)  -->
+
+
+
+<!-- Official PyTorch implementation of [Grounding DINO](https://arxiv.org/abs/2303.05499), a stronger open-set object detector. Code is available now! -->
+
+
+## :bulb: Highlight
+
+- **Open-Set Detection.** Detect **everything** with language!
+- **High Performancce.** COCO zero-shot **52.5 AP** (training without COCO data!). COCO fine-tune **63.0 AP**.
+- **Flexible.** Collaboration with Stable Diffusion for Image Editting.
+
+
+
+
+## :fire: News
+- **`2023/07/18`**: We release [Semantic-SAM](https://github.com/UX-Decoder/Semantic-SAM), a universal image segmentation model to enable segment and recognize anything at any desired granularity. **Code** and **checkpoint** are available!
+- **`2023/06/17`**: We provide an example to evaluate Grounding DINO on COCO zero-shot performance.
+- **`2023/04/15`**: Refer to [CV in the Wild Readings](https://github.com/Computer-Vision-in-the-Wild/CVinW_Readings) for those who are interested in open-set recognition!
+- **`2023/04/08`**: We release [demos](demo/image_editing_with_groundingdino_gligen.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [GLIGEN](https://github.com/gligen/GLIGEN)  for more controllable image editings.
+- **`2023/04/08`**: We release [demos](demo/image_editing_with_groundingdino_stablediffusion.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [Stable Diffusion](https://github.com/Stability-AI/StableDiffusion) for image editings.
+- **`2023/04/06`**: We build a new demo by marrying GroundingDINO with [Segment-Anything](https://github.com/facebookresearch/segment-anything) named **[Grounded-Segment-Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything)** aims to support segmentation in GroundingDINO.
+- **`2023/03/28`**: A YouTube [video](https://youtu.be/cMa77r3YrDk) about Grounding DINO and basic object detection prompt engineering. [[SkalskiP](https://github.com/SkalskiP)]
+- **`2023/03/28`**: Add a [demo](https://huggingface.co/spaces/ShilongLiu/Grounding_DINO_demo) on Hugging Face Space!
+- **`2023/03/27`**: Support CPU-only mode. Now the model can run on machines without GPUs.
+- **`2023/03/25`**: A [demo](https://colab.research.google.com/github/roboflow-ai/notebooks/blob/main/notebooks/zero-shot-object-detection-with-grounding-dino.ipynb) for Grounding DINO is available at Colab. [[SkalskiP](https://github.com/SkalskiP)]
+- **`2023/03/22`**: Code is available Now!
+
+<details open>
+<summary><font size="4">
+Description
+</font></summary>
+ <a href="https://arxiv.org/abs/2303.05499">Paper</a> introduction.
+<img src=".asset/hero_figure.png" alt="ODinW" width="100%">
+Marrying <a href="https://github.com/IDEA-Research/GroundingDINO">Grounding DINO</a> and <a href="https://github.com/gligen/GLIGEN">GLIGEN</a>
+<img src="https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/GD_GLIGEN.png" alt="gd_gligen" width="100%">
+</details>
+
+## :star: Explanations/Tips for Grounding DINO Inputs and Outputs
+- Grounding DINO accepts an `(image, text)` pair as inputs.
+- It outputs `900` (by default) object boxes. Each box has similarity scores across all input words. (as shown in Figures below.)
+- We defaultly choose the boxes whose highest similarities are higher than a `box_threshold`.
+- We extract the words whose similarities are higher than the `text_threshold` as predicted labels.
+- If you want to obtain objects of specific phrases, like the `dogs` in the sentence `two dogs with a stick.`, you can select the boxes with highest text similarities with `dogs` as final outputs. 
+- Note that each word can be split to **more than one** tokens with different tokenlizers. The number of words in a sentence may not equal to the number of text tokens.
+- We suggest separating different category names with `.` for Grounding DINO.
+![model_explain1](.asset/model_explan1.PNG)
+![model_explain2](.asset/model_explan2.PNG)
+
+## :label: TODO 
+
+- [x] Release inference code and demo.
+- [x] Release checkpoints.
+- [x] Grounding DINO with Stable Diffusion and GLIGEN demos.
+- [ ] Release training codes.
+
+## :hammer_and_wrench: Install 
+
+**Note:**
+
+0. If you have a CUDA environment, please make sure the environment variable `CUDA_HOME` is set. It will be compiled under CPU-only mode if no CUDA available.
+
+Please make sure following the installation steps strictly, otherwise the program may produce: 
+```bash
+NameError: name '_C' is not defined
+```
+
+If this happened, please reinstalled the groundingDINO by reclone the git and do all the installation steps again.
+ 
+#### how to check cuda:
+```bash
+echo $CUDA_HOME
+```
+If it print nothing, then it means you haven't set up the path/
+
+Run this so the environment variable will be set under current shell. 
+```bash
+export CUDA_HOME=/path/to/cuda-11.3
+```
+
+Notice the version of cuda should be aligned with your CUDA runtime, for there might exists multiple cuda at the same time. 
+
+If you want to set the CUDA_HOME permanently, store it using:
+
+```bash
+echo 'export CUDA_HOME=/path/to/cuda' >> ~/.bashrc
+```
+after that, source the bashrc file and check CUDA_HOME:
+```bash
+source ~/.bashrc
+echo $CUDA_HOME
+```
+
+In this example, /path/to/cuda-11.3 should be replaced with the path where your CUDA toolkit is installed. You can find this by typing **which nvcc** in your terminal:
+
+For instance, 
+if the output is /usr/local/cuda/bin/nvcc, then:
+```bash
+export CUDA_HOME=/usr/local/cuda
+```
+**Installation:**
+
+1.Clone the GroundingDINO repository from GitHub.
+
+```bash
+git clone https://github.com/IDEA-Research/GroundingDINO.git
+```
+
+2. Change the current directory to the GroundingDINO folder.
+
+```bash
+cd GroundingDINO/
+```
+
+3. Install the required dependencies in the current directory.
+
+```bash
+pip install -e .
+```
+
+4. Download pre-trained model weights.
+
+```bash
+mkdir weights
+cd weights
+wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth
+cd ..
+```
+
+## :arrow_forward: Demo
+Check your GPU ID (only if you're using a GPU)
+
+```bash
+nvidia-smi
+```
+Replace `{GPU ID}`, `image_you_want_to_detect.jpg`, and `"dir you want to save the output"` with appropriate values in the following command
+```bash
+CUDA_VISIBLE_DEVICES={GPU ID} python demo/inference_on_a_image.py \
+-c groundingdino/config/GroundingDINO_SwinT_OGC.py \
+-p weights/groundingdino_swint_ogc.pth \
+-i image_you_want_to_detect.jpg \
+-o "dir you want to save the output" \
+-t "chair"
+ [--cpu-only] # open it for cpu mode
+```
+
+If you would like to specify the phrases to detect, here is a demo:
+```bash
+CUDA_VISIBLE_DEVICES={GPU ID} python demo/inference_on_a_image.py \
+-c groundingdino/config/GroundingDINO_SwinT_OGC.py \
+-p ./groundingdino_swint_ogc.pth \
+-i .asset/cat_dog.jpeg \
+-o logs/1111 \
+-t "There is a cat and a dog in the image ." \
+--token_spans "[[[9, 10], [11, 14]], [[19, 20], [21, 24]]]"
+ [--cpu-only] # open it for cpu mode
+```
+The token_spans specify the start and end positions of a phrases. For example, the first phrase is `[[9, 10], [11, 14]]`. `"There is a cat and a dog in the image ."[9:10] = 'a'`, `"There is a cat and a dog in the image ."[11:14] = 'cat'`. Hence it refers to the phrase `a cat` . Similarly, the `[[19, 20], [21, 24]]` refers to the phrase `a dog`.
+
+See the `demo/inference_on_a_image.py` for more details.
+
+**Running with Python:**
+
+```python
+from groundingdino.util.inference import load_model, load_image, predict, annotate
+import cv2
+
+model = load_model("groundingdino/config/GroundingDINO_SwinT_OGC.py", "weights/groundingdino_swint_ogc.pth")
+IMAGE_PATH = "weights/dog-3.jpeg"
+TEXT_PROMPT = "chair . person . dog ."
+BOX_TRESHOLD = 0.35
+TEXT_TRESHOLD = 0.25
+
+image_source, image = load_image(IMAGE_PATH)
+
+boxes, logits, phrases = predict(
+    model=model,
+    image=image,
+    caption=TEXT_PROMPT,
+    box_threshold=BOX_TRESHOLD,
+    text_threshold=TEXT_TRESHOLD
+)
+
+annotated_frame = annotate(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)
+cv2.imwrite("annotated_image.jpg", annotated_frame)
+```
+**Web UI**
+
+We also provide a demo code to integrate Grounding DINO with Gradio Web UI. See the file `demo/gradio_app.py` for more details.
+
+**Notebooks**
+
+- We release [demos](demo/image_editing_with_groundingdino_gligen.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [GLIGEN](https://github.com/gligen/GLIGEN)  for more controllable image editings.
+- We release [demos](demo/image_editing_with_groundingdino_stablediffusion.ipynb) to combine [Grounding DINO](https://arxiv.org/abs/2303.05499) with [Stable Diffusion](https://github.com/Stability-AI/StableDiffusion) for image editings.
+
+## COCO Zero-shot Evaluations
+
+We provide an example to evaluate Grounding DINO zero-shot performance on COCO. The results should be **48.5**.
+
+```bash
+CUDA_VISIBLE_DEVICES=0 \
+python demo/test_ap_on_coco.py \
+ -c groundingdino/config/GroundingDINO_SwinT_OGC.py \
+ -p weights/groundingdino_swint_ogc.pth \
+ --anno_path /path/to/annoataions/ie/instances_val2017.json \
+ --image_dir /path/to/imagedir/ie/val2017
+```
+
+
+## :luggage: Checkpoints
+
+<!-- insert a table -->
+<table>
+  <thead>
+    <tr style="text-align: right;">
+      <th></th>
+      <th>name</th>
+      <th>backbone</th>
+      <th>Data</th>
+      <th>box AP on COCO</th>
+      <th>Checkpoint</th>
+      <th>Config</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th>1</th>
+      <td>GroundingDINO-T</td>
+      <td>Swin-T</td>
+      <td>O365,GoldG,Cap4M</td>
+      <td>48.4 (zero-shot) / 57.2 (fine-tune)</td>
+      <td><a href="https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth">GitHub link</a> | <a href="https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/groundingdino_swint_ogc.pth">HF link</a></td>
+      <td><a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/groundingdino/config/GroundingDINO_SwinT_OGC.py">link</a></td>
+    </tr>
+    <tr>
+      <th>2</th>
+      <td>GroundingDINO-B</td>
+      <td>Swin-B</td>
+      <td>COCO,O365,GoldG,Cap4M,OpenImage,ODinW-35,RefCOCO</td>
+      <td>56.7 </td>
+      <td><a href="https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha2/groundingdino_swinb_cogcoor.pth">GitHub link</a>  | <a href="https://huggingface.co/ShilongLiu/GroundingDINO/resolve/main/groundingdino_swinb_cogcoor.pth">HF link</a> 
+      <td><a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/groundingdino/config/GroundingDINO_SwinB.cfg.py">link</a></td>
+    </tr>
+  </tbody>
+</table>
+
+## :medal_military: Results
+
+<details open>
+<summary><font size="4">
+COCO Object Detection Results
+</font></summary>
+<img src=".asset/COCO.png" alt="COCO" width="100%">
+</details>
+
+<details open>
+<summary><font size="4">
+ODinW Object Detection Results
+</font></summary>
+<img src=".asset/ODinW.png" alt="ODinW" width="100%">
+</details>
+
+<details open>
+<summary><font size="4">
+Marrying Grounding DINO with <a href="https://github.com/Stability-AI/StableDiffusion">Stable Diffusion</a> for Image Editing
+</font></summary>
+See our example <a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/demo/image_editing_with_groundingdino_stablediffusion.ipynb">notebook</a> for more details.
+<img src=".asset/GD_SD.png" alt="GD_SD" width="100%">
+</details>
+
+
+<details open>
+<summary><font size="4">
+Marrying Grounding DINO with <a href="https://github.com/gligen/GLIGEN">GLIGEN</a> for more Detailed Image Editing.
+</font></summary>
+See our example <a href="https://github.com/IDEA-Research/GroundingDINO/blob/main/demo/image_editing_with_groundingdino_gligen.ipynb">notebook</a> for more details.
+<img src=".asset/GD_GLIGEN.png" alt="GD_GLIGEN" width="100%">
+</details>
+
+## :sauropod: Model: Grounding DINO
+
+Includes: a text backbone, an image backbone, a feature enhancer, a language-guided query selection, and a cross-modality decoder.
+
+![arch](.asset/arch.png)
+
+
+## :hearts: Acknowledgement
+
+Our model is related to [DINO](https://github.com/IDEA-Research/DINO) and [GLIP](https://github.com/microsoft/GLIP). Thanks for their great work!
+
+We also thank great previous work including DETR, Deformable DETR, SMCA, Conditional DETR, Anchor DETR, Dynamic DETR, DAB-DETR, DN-DETR, etc. More related work are available at [Awesome Detection Transformer](https://github.com/IDEACVR/awesome-detection-transformer). A new toolbox [detrex](https://github.com/IDEA-Research/detrex) is available as well.
+
+Thanks [Stable Diffusion](https://github.com/Stability-AI/StableDiffusion) and [GLIGEN](https://github.com/gligen/GLIGEN) for their awesome models.
+
+
+## :black_nib: Citation
+
+If you find our work helpful for your research, please consider citing the following BibTeX entry.   
+
+```bibtex
+@article{liu2023grounding,
+  title={Grounding dino: Marrying dino with grounded pre-training for open-set object detection},
+  author={Liu, Shilong and Zeng, Zhaoyang and Ren, Tianhe and Li, Feng and Zhang, Hao and Yang, Jie and Li, Chunyuan and Yang, Jianwei and Su, Hang and Zhu, Jun and others},
+  journal={arXiv preprint arXiv:2303.05499},
+  year={2023}
+}
+```
+
+
+
+

approach/ovod/GroundingDINO/gdino.py

@@ -0,0 +1,23 @@
+from groundingdino.util.inference import load_model, load_image, predict, annotate
+import cv2
+
+model = load_model("./groundingdino/config/GroundingDINO_SwinT_OGC.py", "weights/groundingdino_swint_ogc.pth")
+
+def peform_ovod(image_path, text_prompt, box_threshold, text_threshold):
+    image_path = "weights/dog.jpg"
+    text_prompt = "chair . person . dog ."
+    box_threshold = 0.35
+    text_threshold = 0.25
+
+    image_source, image = load_image(image_path)
+
+    boxes, logits, phrases = predict(
+        model=model,
+        image=image,
+        caption=text_prompt,
+        box_threshold=box_threshold,
+        text_threshold=text_threshold
+    )
+
+    annotated_frame = annotate(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)
+    cv2.imwrite("annotated_image.jpg", annotated_frame)

approach/ovod/GroundingDINO/requirements.txt

@@ -0,0 +1,10 @@
+torch
+torchvision
+transformers
+addict
+yapf
+timm
+numpy
+opencv-python
+supervision==0.6.0
+pycocotools

approach/ovod/GroundingDINO/setup.py

@@ -0,0 +1,208 @@
+# coding=utf-8
+# Copyright 2022 The IDEA Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ------------------------------------------------------------------------------------------------
+# Modified from
+# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/setup.py
+# https://github.com/facebookresearch/detectron2/blob/main/setup.py
+# https://github.com/open-mmlab/mmdetection/blob/master/setup.py
+# https://github.com/Oneflow-Inc/libai/blob/main/setup.py
+# ------------------------------------------------------------------------------------------------
+
+import glob
+import os
+import subprocess
+
+import torch
+from setuptools import find_packages, setup
+from torch.utils.cpp_extension import CUDA_HOME, CppExtension, CUDAExtension
+
+# groundingdino version info
+version = "0.1.0"
+package_name = "groundingdino"
+cwd = os.path.dirname(os.path.abspath(__file__))
+
+
+sha = "Unknown"
+try:
+    sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd).decode("ascii").strip()
+except Exception:
+    pass
+
+
+def write_version_file():
+    version_path = os.path.join(cwd, "groundingdino", "version.py")
+    with open(version_path, "w") as f:
+        f.write(f"__version__ = '{version}'\n")
+        # f.write(f"git_version = {repr(sha)}\n")
+
+
+requirements = ["torch", "torchvision"]
+
+torch_ver = [int(x) for x in torch.__version__.split(".")[:2]]
+
+
+def get_extensions():
+    this_dir = os.path.dirname(os.path.abspath(__file__))
+    extensions_dir = os.path.join(this_dir, "groundingdino", "models", "GroundingDINO", "csrc")
+
+    main_source = os.path.join(extensions_dir, "vision.cpp")
+    sources = glob.glob(os.path.join(extensions_dir, "**", "*.cpp"))
+    source_cuda = glob.glob(os.path.join(extensions_dir, "**", "*.cu")) + glob.glob(
+        os.path.join(extensions_dir, "*.cu")
+    )
+
+    sources = [main_source] + sources
+
+    extension = CppExtension
+
+    extra_compile_args = {"cxx": []}
+    define_macros = []
+
+    if CUDA_HOME is not None and (torch.cuda.is_available() or "TORCH_CUDA_ARCH_LIST" in os.environ):
+        print("Compiling with CUDA")
+        extension = CUDAExtension
+        sources += source_cuda
+        define_macros += [("WITH_CUDA", None)]
+        extra_compile_args["nvcc"] = [
+            "-DCUDA_HAS_FP16=1",
+            "-D__CUDA_NO_HALF_OPERATORS__",
+            "-D__CUDA_NO_HALF_CONVERSIONS__",
+            "-D__CUDA_NO_HALF2_OPERATORS__",
+        ]
+    else:
+        print("Compiling without CUDA")
+        define_macros += [("WITH_HIP", None)]
+        extra_compile_args["nvcc"] = []
+        return None
+
+    sources = [os.path.join(extensions_dir, s) for s in sources]
+    include_dirs = [extensions_dir]
+
+    ext_modules = [
+        extension(
+            "groundingdino._C",
+            sources,
+            include_dirs=include_dirs,
+            define_macros=define_macros,
+            extra_compile_args=extra_compile_args,
+        )
+    ]
+
+    return ext_modules
+
+
+def parse_requirements(fname="requirements.txt", with_version=True):
+    """Parse the package dependencies listed in a requirements file but strips
+    specific versioning information.
+
+    Args:
+        fname (str): path to requirements file
+        with_version (bool, default=False): if True include version specs
+
+    Returns:
+        List[str]: list of requirements items
+
+    CommandLine:
+        python -c "import setup; print(setup.parse_requirements())"
+    """
+    import re
+    import sys
+    from os.path import exists
+
+    require_fpath = fname
+
+    def parse_line(line):
+        """Parse information from a line in a requirements text file."""
+        if line.startswith("-r "):
+            # Allow specifying requirements in other files
+            target = line.split(" ")[1]
+            for info in parse_require_file(target):
+                yield info
+        else:
+            info = {"line": line}
+            if line.startswith("-e "):
+                info["package"] = line.split("#egg=")[1]
+            elif "@git+" in line:
+                info["package"] = line
+            else:
+                # Remove versioning from the package
+                pat = "(" + "|".join([">=", "==", ">"]) + ")"
+                parts = re.split(pat, line, maxsplit=1)
+                parts = [p.strip() for p in parts]
+
+                info["package"] = parts[0]
+                if len(parts) > 1:
+                    op, rest = parts[1:]
+                    if ";" in rest:
+                        # Handle platform specific dependencies
+                        # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies
+                        version, platform_deps = map(str.strip, rest.split(";"))
+                        info["platform_deps"] = platform_deps
+                    else:
+                        version = rest  # NOQA
+                    info["version"] = (op, version)
+            yield info
+
+    def parse_require_file(fpath):
+        with open(fpath, "r") as f:
+            for line in f.readlines():
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    for info in parse_line(line):
+                        yield info
+
+    def gen_packages_items():
+        if exists(require_fpath):
+            for info in parse_require_file(require_fpath):
+                parts = [info["package"]]
+                if with_version and "version" in info:
+                    parts.extend(info["version"])
+                if not sys.version.startswith("3.4"):
+                    # apparently package_deps are broken in 3.4
+                    platform_deps = info.get("platform_deps")
+                    if platform_deps is not None:
+                        parts.append(";" + platform_deps)
+                item = "".join(parts)
+                yield item
+
+    packages = list(gen_packages_items())
+    return packages
+
+
+if __name__ == "__main__":
+    print(f"Building wheel {package_name}-{version}")
+
+    with open("LICENSE", "r", encoding="utf-8") as f:
+        license = f.read()
+
+    write_version_file()
+
+    setup(
+        name="groundingdino",
+        version="0.1.0",
+        author="International Digital Economy Academy, Shilong Liu",
+        url="https://github.com/IDEA-Research/GroundingDINO",
+        description="open-set object detector",
+        license=license,
+        install_requires=parse_requirements("requirements.txt"),
+        packages=find_packages(
+            exclude=(
+                "configs",
+                "tests",
+            )
+        ),
+        ext_modules=get_extensions(),
+        cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension},
+    )

approach/ovod/GroundingDINO/test.ipynb

@@ -0,0 +1,55 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from groundingdino.util.inference import load_model, load_image, predict, annotate\n",
+    "import cv2\n",
+    "\n",
+    "model = load_model(\"groundingdino/config/GroundingDINO_SwinT_OGC.py\", \"../04-06-segment-anything/weights/groundingdino_swint_ogc.pth\")\n",
+    "IMAGE_PATH = \".asset/cat_dog.jpeg\"\n",
+    "TEXT_PROMPT = \"chair . person . dog .\"\n",
+    "BOX_TRESHOLD = 0.35\n",
+    "TEXT_TRESHOLD = 0.25\n",
+    "\n",
+    "image_source, image = load_image(IMAGE_PATH)\n",
+    "\n",
+    "boxes, logits, phrases = predict(\n",
+    "    model=model,\n",
+    "    image=image,\n",
+    "    caption=TEXT_PROMPT,\n",
+    "    box_threshold=BOX_TRESHOLD,\n",
+    "    text_threshold=TEXT_TRESHOLD\n",
+    ")\n",
+    "\n",
+    "annotated_frame = annotate(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)\n",
+    "cv2.imwrite(\"annotated_image.jpg\", annotated_frame)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

approach/ovod/d-cube/.gitignore

@@ -0,0 +1,164 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+.vscode/*
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# mac system
+*.DS_Store

approach/ovod/d-cube/README.md

@@ -0,0 +1,134 @@
+<!-- PROJECT LOGO -->
+<br />
+<p align="center">
+  <a href="#">
+<img src=".assets/d-cube_logo.png" alt="Logo" width="310"></a>
+  <h4 align="center">A detection/segmentation dataset with class names characterized by intricate and flexible expressions</h4>
+    <p align="center">
+    The repo is the toolbox for <b>D<sup>3</sup></b>
+    <br />
+    <a href="doc.md"><strong> [Doc 📚]</strong></a>
+    <!-- <a href="https://huggingface.co/datasets/zbrl/d-cube"><strong> [HuggingFace 🤗]</strong></a> -->
+    <a href="https://arxiv.org/abs/2307.12813"><strong> [Paper (DOD) 📄] </strong></a>
+    <a href="https://arxiv.org/abs/2305.12452"><strong> [Paper (GRES) 📄] </strong></a>
+    <a href="https://github.com/Charles-Xie/awesome-described-object-detection"><strong> [Awesome-DOD 🕶️] </strong></a>
+    <br />
+  </p>
+</p>
+
+***
+Description Detection Dataset ($D^3$, /dikju:b/) is an attempt at creating a next-generation object detection dataset. Unlike traditional detection datasets, the class names of the objects are no longer simple nouns or noun phrases, but rather complex and descriptive, such as `a dog not being held by a leash`. For each image in the dataset, any object that matches the description is annotated. The dataset provides annotations such as bounding boxes and finely crafted instance masks. We believe it will contribute to computer vision and vision-language communities.
+
+
+
+# News
+- [02/14/2024] Evaluation on several SOTA methods (SPHNX (the first MLLM evaluated!), G-DINO, UNINEXT, etc.) are released, together with a [leaderboard](https://github.com/shikras/d-cube/tree/main/eval_sota) for $D^3$. :fire::fire:
+
+- [10/12/2023] We released an [awesome-described-object-detection](https://github.com/Charles-Xie/awesome-described-object-detection) list to collect and track related works.
+
+- [09/22/2023] Our DOD [paper](https://arxiv.org/abs/2307.12813) just got accepted by NeurIPS 2023! :fire:
+
+- [07/25/2023] This toolkit is available on PyPI now. You can install this repo with `pip install ddd-dataset`.
+
+- [07/25/2023] The [paper preprint](https://arxiv.org/abs/2307.12813) introducing the DOD task and the $D^3$ dataset, is available on arxiv. Check it out!
+
+- [07/18/2023] We have released our Description Detection Dataset ($D^3$) and the first version of $D^3$ toolbox. You can download it now for your project.
+
+- [07/14/2023] Our GRES [paper](https://arxiv.org/abs/2305.12452) has been accepted by ICCV 2023.
+
+
+
+# Contents
+- [Dataset Highlight](#task-and-dataset-highlight)
+- [Download](#download)
+- [Installation](#installation)
+- [Usage](#usage)
+
+
+
+# Task and Dataset Highlight
+
+The $D^3$ dataset is meant for the Described Object Detection (DOD) task. In the image below we show the difference between Referring Expression Comprehension (REC), Object Detection/Open-Vocabulary Detection (OVD) and Described Object Detection (DOD). OVD detect object based on category name, and each category can have zero to multiple instances; REC grounds one region based on a language description, whether the object truly exits or not; DOD detect all instances on each image in the dataset, based on a flexible reference. Related works are tracked in the [awesome-DOD](https://github.com/Charles-Xie/awesome-described-object-detection) list.
+
+![Dataset Highlight](.assets/teaser.png "Highlight of the task & dataset")
+
+For more information on the characteristics of this dataset, please refer to our paper.
+
+
+
+# Download
+Currently we host the $D^3$ dataset on cloud drives. You can download the dataset from [Google Drive](https://drive.google.com/drive/folders/11kfY12NzKPwsliLEcIYki1yUqt7PbMEi?usp=sharing) or [Baidu Pan]().
+
+After downloading the `d3_images.zip` (images in the dataset), `d3_pkl.zip` (dataset information for this toolkit) and `d3_json.zip` (annotation for evaluation), please extract these 3 zip files to your custom `IMG_ROOT`, `PKL_PATH` and `JSON_ANNO_PATH` directory. These paths will be used when you perform inference or evaluation on this dataset.
+
+
+
+# Installation
+
+## Prerequisites
+This toolkit requires a few python packages like `numpy` and `pycocotools`. Other packages like `matplotlib` and `opencv-python` may also be required if you want to utilize the visualization scripts.
+
+<!-- There are three ways to install $D^3$ toolbox, and the third one (with huggingface) is currently in the works and will be available soon. -->
+
+There are multiple ways to install $D^3$ toolbox, as listed below:
+
+
+## Install with pip
+```bash
+pip install ddd-dataset
+```
+
+## Install from source
+```bash
+git clone https://github.com/shikra/d-cube.git
+# option 1: install it as a python package
+cd d-cube
+python -m pip install .
+# done
+
+# option 2: just put the d-cube/d_cube directory in the root directory of your local repository
+```
+
+<!-- ## Via HuggingFace Datasets 🤗
+```bash
+coming soon
+``` -->
+
+
+
+# Usage
+Please refer to the [documentation 📚](doc.md) for more details.
+Our toolbox is similar to [cocoapi](https://github.com/cocodataset/cocoapi) in style.
+
+Here is a quick example of how to use $D^3$.
+```python
+from d_cube import D3
+d3 = D3(IMG_ROOT, PKL_ANNO_PATH)
+all_img_ids = d3.get_img_ids()  # get the image ids in the dataset
+all_img_info = d3.load_imgs(all_img_ids)  # load images by passing a list of some image ids
+img_path = all_img_info[0]["file_name"]  # obtain one image path so you can load it and inference
+```
+
+Some frequently asked questions are answered in [this Q&A file](./qa.md).
+
+# Citation
+
+If you use our $D^3$ dataset, this toolbox, or otherwise find our work valuable, please cite [our paper](https://arxiv.org/abs/2307.12813):
+
+```bibtex
+@inproceedings{xie2023DOD,
+  title={Described Object Detection: Liberating Object Detection with Flexible Expressions},
+  author={Xie, Chi and Zhang, Zhao and Wu, Yixuan and Zhu, Feng and Zhao, Rui and Liang, Shuang},
+  booktitle={Thirty-seventh Conference on Neural Information Processing Systems (NeurIPS)},
+  year={2023}
+}
+
+@inproceedings{wu2023gres,
+  title={Advancing Referring Expression Segmentation Beyond Single Image},
+  author={Wu, Yixuan and Zhang, Zhao and Xie, Chi and Zhu, Feng and Zhao, Rui},
+  booktitle={International Conference on Computer Vision (ICCV)},
+  year={2023}
+}
+```
+
+More works related to Described Object Detection are tracked in this list: [awesome-described-object-detection](https://github.com/Charles-Xie/awesome-described-object-detection).

approach/ovod/d-cube/qa.md

@@ -0,0 +1,24 @@
+# Frequently Asked Questions
+
+Q:
+What's the difference between Intra-Group and Inter-Group setting in [the DOD paper](https://arxiv.org/abs/2307.12813), and how to set them?
+
+A:
+Please see [this explanation in the document](./doc.md#intra--or-inter-group-settings).
+
+
+
+Q:
+What's the meaning of and difference between FULL, PRES, and ABS?
+
+A:
+Please see [this explanation in the document](./doc.md#full-pres-and-abs).
+
+
+
+Q:
+How do I perform a visualization of ground truth or prediction on a image?
+
+A:
+You can use `d3.get_anno_ids` function and pass the `img_id` you choose as parameter to get the annotation ids for a image.
+After this, you can obtain the annotation details (class ids, bboxes) with `d3.load_annos`.

approach/ovod/d-cube/requirements.txt

@@ -0,0 +1,4 @@
+numpy
+pycocotools
+opencv-python
+matplotlib

approach/ovod/d-cube/setup.py

@@ -0,0 +1,30 @@
+import setuptools
+
+setuptools.setup(
+    name='ddd-dataset',
+    version='0.1.1',
+    author='Chi Xie',
+    author_email='chixie.personal@gmail.com',
+    description='Toolkit for Description Detection Dataset ($D^3$)',
+    long_description='Toolkit for Description Detection Dataset ($D^3$): A detection dataset with class names characterized by intricate and flexible expressions, for the Described Object Detection (DOD) task.',
+    long_description_content_type='text/markdown',
+    license='CC BY-NC 4.0',
+    packages=['d_cube'],
+    package_dir={"d_cube": "d_cube"},
+    url='https://github.com/shikras/d-cube',
+    project_urls={
+        "Bug Tracker": "https://github.com/shikras/d-cube/issues",
+    },
+    install_requires=['numpy', 'pycocotools', 'opencv-python', 'matplotlib'],
+
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Intended Audience :: Science/Research',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Education',
+        'Operating System :: MacOS',
+        'Operating System :: Microsoft :: Windows',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python :: 3',
+    ],
+)

approach/ovod/detectron2/.clang-format

@@ -0,0 +1,85 @@
+AccessModifierOffset: -1
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   false
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: false
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ForEachMacros:   [ FOR_EACH, FOR_EACH_R, FOR_EACH_RANGE, ]
+IncludeCategories:
+  - Regex:           '^<.*\.h(pp)?>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never

approach/ovod/detectron2/.flake8

@@ -0,0 +1,15 @@
+# This is an example .flake8 config, used when developing *Black* itself.
+# Keep in sync with setup.cfg which is used for source packages.
+
+[flake8]
+ignore = W503, E203, E221, C901, C408, E741, C407, B017, F811, C101, EXE001, EXE002
+max-line-length = 100
+max-complexity = 18
+select = B,C,E,F,W,T4,B9
+exclude = build
+per-file-ignores =
+  **/__init__.py:F401,F403,E402
+  **/configs/**.py:F401,E402
+  configs/**.py:F401,E402
+  **/tests/config/**.py:F401,E402
+  tests/config/**.py:F401,E402

approach/ovod/detectron2/.gitignore

@@ -0,0 +1,53 @@
+# output dir
+output
+instant_test_output
+inference_test_output
+
+
+*.png
+*.json
+*.diff
+*.jpg
+!/projects/DensePose/doc/images/*.jpg
+
+# compilation and distribution
+__pycache__
+_ext
+*.pyc
+*.pyd
+*.so
+*.dll
+*.egg-info/
+build/
+dist/
+wheels/
+
+# pytorch/python/numpy formats
+*.pth
+*.pkl
+*.npy
+*.ts
+model_ts*.txt
+
+# ipython/jupyter notebooks
+*.ipynb
+**/.ipynb_checkpoints/
+
+# Editor temporaries
+*.swn
+*.swo
+*.swp
+*~
+
+# editor settings
+.idea
+.vscode
+_darcs
+
+# project dirs
+/detectron2/model_zoo/configs
+/datasets/*
+!/datasets/*.*
+/projects/*/datasets
+/models
+/snippet

approach/ovod/detectron2/GETTING_STARTED.md

@@ -0,0 +1,79 @@
+## Getting Started with Detectron2
+
+This document provides a brief intro of the usage of builtin command-line tools in detectron2.
+
+For a tutorial that involves actual coding with the API,
+see our [Colab Notebook](https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5)
+which covers how to run inference with an
+existing model, and how to train a builtin model on a custom dataset.
+
+
+### Inference Demo with Pre-trained Models
+
+1. Pick a model and its config file from
+  [model zoo](MODEL_ZOO.md),
+  for example, `mask_rcnn_R_50_FPN_3x.yaml`.
+2. We provide `demo.py` that is able to demo builtin configs. Run it with:
+```
+cd demo/
+python demo.py --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \
+  --input input1.jpg input2.jpg \
+  [--other-options]
+  --opts MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl
+```
+The configs are made for training, therefore we need to specify `MODEL.WEIGHTS` to a model from model zoo for evaluation.
+This command will run the inference and show visualizations in an OpenCV window.
+
+For details of the command line arguments, see `demo.py -h` or look at its source code
+to understand its behavior. Some common arguments are:
+* To run __on your webcam__, replace `--input files` with `--webcam`.
+* To run __on a video__, replace `--input files` with `--video-input video.mp4`.
+* To run __on cpu__, add `MODEL.DEVICE cpu` after `--opts`.
+* To save outputs to a directory (for images) or a file (for webcam or video), use `--output`.
+
+
+### Training & Evaluation in Command Line
+
+We provide two scripts in "tools/plain_train_net.py" and "tools/train_net.py",
+that are made to train all the configs provided in detectron2. You may want to
+use it as a reference to write your own training script.
+
+Compared to "train_net.py", "plain_train_net.py" supports fewer default
+features. It also includes fewer abstraction, therefore is easier to add custom
+logic.
+
+To train a model with "train_net.py", first
+setup the corresponding datasets following
+[datasets/README.md](./datasets/README.md),
+then run:
+```
+cd tools/
+./train_net.py --num-gpus 8 \
+  --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml
+```
+
+The configs are made for 8-GPU training.
+To train on 1 GPU, you may need to [change some parameters](https://arxiv.org/abs/1706.02677), e.g.:
+```
+./train_net.py \
+  --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+  --num-gpus 1 SOLVER.IMS_PER_BATCH 2 SOLVER.BASE_LR 0.0025
+```
+
+To evaluate a model's performance, use
+```
+./train_net.py \
+  --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+  --eval-only MODEL.WEIGHTS /path/to/checkpoint_file
+```
+For more options, see `./train_net.py -h`.
+
+### Use Detectron2 APIs in Your Code
+
+See our [Colab Notebook](https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5)
+to learn how to use detectron2 APIs to:
+1. run inference with an existing model
+2. train a builtin model on a custom dataset
+
+See [detectron2/projects](https://github.com/facebookresearch/detectron2/tree/main/projects)
+for more ways to build your project on detectron2.

approach/ovod/detectron2/INSTALL.md

@@ -0,0 +1,261 @@
+## Installation
+
+### Requirements
+- Linux or macOS with Python ≥ 3.7
+- PyTorch ≥ 1.8 and [torchvision](https://github.com/pytorch/vision/) that matches the PyTorch installation.
+  Install them together at [pytorch.org](https://pytorch.org) to make sure of this
+- OpenCV is optional but needed by demo and visualization
+
+
+### Build Detectron2 from Source
+
+gcc & g++ ≥ 5.4 are required. [ninja](https://ninja-build.org/) is optional but recommended for faster build.
+After having them, run:
+```
+python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'
+# (add --user if you don't have permission)
+
+# Or, to install it from a local clone:
+git clone https://github.com/facebookresearch/detectron2.git
+python -m pip install -e detectron2
+
+# On macOS, you may need to prepend the above commands with a few environment variables:
+CC=clang CXX=clang++ ARCHFLAGS="-arch x86_64" python -m pip install ...
+```
+
+To __rebuild__ detectron2 that's built from a local clone, use `rm -rf build/ **/*.so` to clean the
+old build first. You often need to rebuild detectron2 after reinstalling PyTorch.
+
+### Install Pre-Built Detectron2 (Linux only)
+
+Choose from this table to install [v0.6 (Oct 2021)](https://github.com/facebookresearch/detectron2/releases):
+
+<table class="docutils"><tbody><th width="80"> CUDA </th><th valign="bottom" align="left" width="100">torch 1.10</th><th valign="bottom" align="left" width="100">torch 1.9</th><th valign="bottom" align="left" width="100">torch 1.8</th> <tr><td align="left">11.3</td><td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu113/torch1.10/index.html
+</code></pre> </details> </td> <td align="left"> </td> <td align="left"> </td> </tr> <tr><td align="left">11.1</td><td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu111/torch1.10/index.html
+</code></pre> </details> </td> <td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu111/torch1.9/index.html
+</code></pre> </details> </td> <td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu111/torch1.8/index.html
+</code></pre> </details> </td> </tr> <tr><td align="left">10.2</td><td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.10/index.html
+</code></pre> </details> </td> <td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.9/index.html
+</code></pre> </details> </td> <td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.8/index.html
+</code></pre> </details> </td> </tr> <tr><td align="left">10.1</td><td align="left"> </td> <td align="left"> </td> <td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/torch1.8/index.html
+</code></pre> </details> </td> </tr> <tr><td align="left">cpu</td><td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cpu/torch1.10/index.html
+</code></pre> </details> </td> <td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cpu/torch1.9/index.html
+</code></pre> </details> </td> <td align="left"><details><summary> install </summary><pre><code>python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cpu/torch1.8/index.html
+</code></pre> </details> </td> </tr></tbody></table>
+
+Note that:
+1. The pre-built packages have to be used with corresponding version of CUDA and the official package of PyTorch.
+   Otherwise, please build detectron2 from source.
+2. New packages are released every few months. Therefore, packages may not contain latest features in the main
+   branch and may not be compatible with the main branch of a research project that uses detectron2
+   (e.g. those in [projects](projects)).
+
+### Common Installation Issues
+
+Click each issue for its solutions:
+
+<details>
+<summary>
+Undefined symbols that looks like "TH..","at::Tensor...","torch..."
+</summary>
+<br/>
+
+This usually happens when detectron2 or torchvision is not
+compiled with the version of PyTorch you're running.
+
+If the error comes from a pre-built torchvision, uninstall torchvision and pytorch and reinstall them
+following [pytorch.org](http://pytorch.org). So the versions will match.
+
+If the error comes from a pre-built detectron2, check [release notes](https://github.com/facebookresearch/detectron2/releases),
+uninstall and reinstall the correct pre-built detectron2 that matches pytorch version.
+
+If the error comes from detectron2 or torchvision that you built manually from source,
+remove files you built (`build/`, `**/*.so`) and rebuild it so it can pick up the version of pytorch currently in your environment.
+
+If the above instructions do not resolve this problem, please provide an environment (e.g. a dockerfile) that can reproduce the issue.
+</details>
+
+<details>
+<summary>
+Missing torch dynamic libraries, OR segmentation fault immediately when using detectron2.
+</summary>
+This usually happens when detectron2 or torchvision is not
+compiled with the version of PyTorch you're running. See the previous common issue for the solution.
+</details>
+
+<details>
+<summary>
+Undefined C++ symbols (e.g. "GLIBCXX..") or C++ symbols not found.
+</summary>
+<br/>
+Usually it's because the library is compiled with a newer C++ compiler but run with an old C++ runtime.
+
+This often happens with old anaconda.
+It may help to run `conda update libgcc` to upgrade its runtime.
+
+The fundamental solution is to avoid the mismatch, either by compiling using older version of C++
+compiler, or run the code with proper C++ runtime.
+To run the code with a specific C++ runtime, you can use environment variable `LD_PRELOAD=/path/to/libstdc++.so`.
+
+</details>
+
+<details>
+<summary>
+"nvcc not found" or "Not compiled with GPU support" or "Detectron2 CUDA Compiler: not available".
+</summary>
+<br/>
+CUDA is not found when building detectron2.
+You should make sure
+
+```
+python -c 'import torch; from torch.utils.cpp_extension import CUDA_HOME; print(torch.cuda.is_available(), CUDA_HOME)'
+```
+
+print `(True, a directory with cuda)` at the time you build detectron2.
+
+Most models can run inference (but not training) without GPU support. To use CPUs, set `MODEL.DEVICE='cpu'` in the config.
+</details>
+
+<details>
+<summary>
+"invalid device function" or "no kernel image is available for execution".
+</summary>
+<br/>
+Two possibilities:
+
+* You build detectron2 with one version of CUDA but run it with a different version.
+
+  To check whether it is the case,
+  use `python -m detectron2.utils.collect_env` to find out inconsistent CUDA versions.
+  In the output of this command, you should expect "Detectron2 CUDA Compiler", "CUDA_HOME", "PyTorch built with - CUDA"
+  to contain cuda libraries of the same version.
+
+  When they are inconsistent,
+  you need to either install a different build of PyTorch (or build by yourself)
+  to match your local CUDA installation, or install a different version of CUDA to match PyTorch.
+
+* PyTorch/torchvision/Detectron2 is not built for the correct GPU SM architecture (aka. compute capability).
+
+  The architecture included by PyTorch/detectron2/torchvision is available in the "architecture flags" in
+  `python -m detectron2.utils.collect_env`. It must include
+  the architecture of your GPU, which can be found at [developer.nvidia.com/cuda-gpus](https://developer.nvidia.com/cuda-gpus).
+
+  If you're using pre-built PyTorch/detectron2/torchvision, they have included support for most popular GPUs already.
+  If not supported, you need to build them from source.
+
+  When building detectron2/torchvision from source, they detect the GPU device and build for only the device.
+  This means the compiled code may not work on a different GPU device.
+  To recompile them for the correct architecture, remove all installed/compiled files,
+  and rebuild them with the `TORCH_CUDA_ARCH_LIST` environment variable set properly.
+  For example, `export TORCH_CUDA_ARCH_LIST="6.0;7.0"` makes it compile for both P100s and V100s.
+</details>
+
+<details>
+<summary>
+Undefined CUDA symbols; Cannot open libcudart.so
+</summary>
+<br/>
+The version of NVCC you use to build detectron2 or torchvision does
+not match the version of CUDA you are running with.
+This often happens when using anaconda's CUDA runtime.
+
+Use `python -m detectron2.utils.collect_env` to find out inconsistent CUDA versions.
+In the output of this command, you should expect "Detectron2 CUDA Compiler", "CUDA_HOME", "PyTorch built with - CUDA"
+to contain cuda libraries of the same version.
+
+When they are inconsistent,
+you need to either install a different build of PyTorch (or build by yourself)
+to match your local CUDA installation, or install a different version of CUDA to match PyTorch.
+</details>
+
+
+<details>
+<summary>
+C++ compilation errors from NVCC / NVRTC, or "Unsupported gpu architecture"
+</summary>
+<br/>
+A few possibilities:
+
+1. Local CUDA/NVCC version has to match the CUDA version of your PyTorch. Both can be found in `python collect_env.py`
+   (download from [here](./detectron2/utils/collect_env.py)).
+   When they are inconsistent, you need to either install a different build of PyTorch (or build by yourself)
+   to match your local CUDA installation, or install a different version of CUDA to match PyTorch.
+
+2. Local CUDA/NVCC version shall support the SM architecture (a.k.a. compute capability) of your GPU.
+   The capability of your GPU can be found at [developer.nvidia.com/cuda-gpus](https://developer.nvidia.com/cuda-gpus).
+   The capability supported by NVCC is listed at [here](https://gist.github.com/ax3l/9489132).
+   If your NVCC version is too old, this can be workaround by setting environment variable
+   `TORCH_CUDA_ARCH_LIST` to a lower, supported capability.
+
+3. The combination of NVCC and GCC you use is incompatible. You need to change one of their versions.
+   See [here](https://gist.github.com/ax3l/9489132) for some valid combinations.
+   Notably, CUDA<=10.1.105 doesn't support GCC>7.3.
+
+   The CUDA/GCC version used by PyTorch can be found by `print(torch.__config__.show())`.
+
+</details>
+
+
+<details>
+<summary>
+"ImportError: cannot import name '_C'".
+</summary>
+<br/>
+Please build and install detectron2 following the instructions above.
+
+Or, if you are running code from detectron2's root directory, `cd` to a different one.
+Otherwise you may not import the code that you installed.
+</details>
+
+
+<details>
+<summary>
+Any issue on windows.
+</summary>
+<br/>
+
+Detectron2 is continuously built on windows with [CircleCI](https://app.circleci.com/pipelines/github/facebookresearch/detectron2?branch=main).
+However we do not provide official support for it.
+PRs that improves code compatibility on windows are welcome.
+</details>
+
+<details>
+<summary>
+ONNX conversion segfault after some "TraceWarning".
+</summary>
+<br/>
+The ONNX package is compiled with a too old compiler.
+
+Please build and install ONNX from its source code using a compiler
+whose version is closer to what's used by PyTorch (available in `torch.__config__.show()`).
+</details>
+
+
+<details>
+<summary>
+"library not found for -lstdc++" on older version of MacOS
+</summary>
+<br/>
+
+See [this stackoverflow answer](https://stackoverflow.com/questions/56083725/macos-build-issues-lstdc-not-found-while-building-python-package).
+
+</details>
+
+
+### Installation inside specific environments:
+
+* __Colab__: see our [Colab Tutorial](https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5)
+  which has step-by-step instructions.
+
+* __Docker__: The official [Dockerfile](docker) installs detectron2 with a few simple commands.

approach/ovod/detectron2/LICENSE

@@ -0,0 +1,202 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction,
+and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by
+the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all
+other entities that control, are controlled by, or are under common
+control with that entity. For the purposes of this definition,
+"control" means (i) the power, direct or indirect, to cause the
+direction or management of such entity, whether by contract or
+otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity
+exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation
+source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical
+transformation or translation of a Source form, including but
+not limited to compiled object code, generated documentation,
+and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or
+Object form, made available under the License, as indicated by a
+copyright notice that is included in or attached to the work
+(an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object
+form, that is based on (or derived from) the Work and for which the
+editorial revisions, annotations, elaborations, or other modifications
+represent, as a whole, an original work of authorship. For the purposes
+of this License, Derivative Works shall not include works that remain
+separable from, or merely link (or bind by name) to the interfaces of,
+the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including
+the original version of the Work and any modifications or additions
+to that Work or Derivative Works thereof, that is intentionally
+submitted to Licensor for inclusion in the Work by the copyright owner
+or by an individual or Legal Entity authorized to submit on behalf of
+the copyright owner. For the purposes of this definition, "submitted"
+means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems,
+and issue tracking systems that are managed by, or on behalf of, the
+Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise
+designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity
+on behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the
+Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+(except as stated in this section) patent license to make, have made,
+use, offer to sell, sell, import, and otherwise transfer the Work,
+where such license applies only to those patent claims licensable
+by such Contributor that are necessarily infringed by their
+Contribution(s) alone or by combination of their Contribution(s)
+with the Work to which such Contribution(s) was submitted. If You
+institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work
+or a Contribution incorporated within the Work constitutes direct
+or contributory patent infringement, then any patent licenses
+granted to You under this License for that Work shall terminate
+as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+Work or Derivative Works thereof in any medium, with or without
+modifications, and in Source or Object form, provided that You
+meet the following conditions:
+
+(a) You must give any other recipients of the Work or
+Derivative Works a copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices
+stating that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works
+that You distribute, all copyright, patent, trademark, and
+attribution notices from the Source form of the Work,
+excluding those notices that do not pertain to any part of
+the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its
+distribution, then any Derivative Works that You distribute must
+include a readable copy of the attribution notices contained
+within such NOTICE file, excluding those notices that do not
+pertain to any part of the Derivative Works, in at least one
+of the following places: within a NOTICE text file distributed
+as part of the Derivative Works; within the Source form or
+documentation, if provided along with the Derivative Works; or,
+within a display generated by the Derivative Works, if and
+wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and
+do not modify the License. You may add Your own attribution
+notices within Derivative Works that You distribute, alongside
+or as an addendum to the NOTICE text from the Work, provided
+that such additional attribution notices cannot be construed
+as modifying the License.
+
+You may add Your own copyright statement to Your modifications and
+may provide additional or different license terms and conditions
+for use, reproduction, or distribution of Your modifications, or
+for any such Derivative Works as a whole, provided Your use,
+reproduction, and distribution of the Work otherwise complies with
+the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+any Contribution intentionally submitted for inclusion in the Work
+by You to the Licensor shall be under the terms and conditions of
+this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify
+the terms of any separate license agreement you may have executed
+with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+names, trademarks, service marks, or product names of the Licensor,
+except as required for reasonable and customary use in describing the
+origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+agreed to in writing, Licensor provides the Work (and each
+Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied, including, without limitation, any warranties or conditions
+of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any
+risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+whether in tort (including negligence), contract, or otherwise,
+unless required by applicable law (such as deliberate and grossly
+negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special,
+incidental, or consequential damages of any character arising as a
+result of this License or out of the use or inability to use the
+Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all
+other commercial damages or losses), even if such Contributor
+has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+the Work or Derivative Works thereof, You may choose to offer,
+and charge a fee for, acceptance of support, warranty, indemnity,
+or other liability obligations and/or rights consistent with this
+License. However, in accepting such obligations, You may act only
+on Your own behalf and on Your sole responsibility, not on behalf
+of any other Contributor, and only if You agree to indemnify,
+defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason
+of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following
+boilerplate notice, with the fields enclosed by brackets "[]"
+replaced with your own identifying information. (Don't include
+the brackets!)  The text should be enclosed in the appropriate
+comment syntax for the file format. We also recommend that a
+file or class name and description of purpose be included on the
+same "printed page" as the copyright notice for easier
+identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

approach/ovod/detectron2/MODEL_ZOO.md

@@ -0,0 +1,1052 @@
+# Detectron2 Model Zoo and Baselines
+
+## Introduction
+
+This file documents a large collection of baselines trained
+with detectron2 in Sep-Oct, 2019.
+All numbers were obtained on [Big Basin](https://engineering.fb.com/data-center-engineering/introducing-big-basin-our-next-generation-ai-hardware/)
+servers with 8 NVIDIA V100 GPUs & NVLink. The speed numbers are periodically updated with latest PyTorch/CUDA/cuDNN versions.
+You can access these models from code using [detectron2.model_zoo](https://detectron2.readthedocs.io/modules/model_zoo.html) APIs.
+
+In addition to these official baseline models, you can find more models in [projects/](projects/).
+
+#### How to Read the Tables
+* The "Name" column contains a link to the config file. Models can be reproduced using `tools/train_net.py` with the corresponding yaml config file,
+  or `tools/lazyconfig_train_net.py` for python config files.
+* Training speed is averaged across the entire training.
+  We keep updating the speed with latest version of detectron2/pytorch/etc.,
+  so they might be different from the `metrics` file.
+  Training speed for multi-machine jobs is not provided.
+* Inference speed is measured by `tools/train_net.py --eval-only`, or [inference_on_dataset()](https://detectron2.readthedocs.io/modules/evaluation.html#detectron2.evaluation.inference_on_dataset),
+  with batch size 1 in detectron2 directly.
+  Measuring it with custom code may introduce other overhead.
+  Actual deployment in production should in general be faster than the given inference
+  speed due to more optimizations.
+* The *model id* column is provided for ease of reference.
+  To check downloaded file integrity, any model on this page contains its md5 prefix in its file name.
+* Training curves and other statistics can be found in `metrics` for each model.
+
+#### Common Settings for COCO Models
+* All COCO models were trained on `train2017` and evaluated on `val2017`.
+* The default settings are __not directly comparable__ with Detectron's standard settings.
+  For example, our default training data augmentation uses scale jittering in addition to horizontal flipping.
+
+  To make fair comparisons with Detectron's settings, see
+  [Detectron1-Comparisons](configs/Detectron1-Comparisons/) for accuracy comparison,
+  and [benchmarks](https://detectron2.readthedocs.io/notes/benchmarks.html)
+  for speed comparison.
+* For Faster/Mask R-CNN, we provide baselines based on __3 different backbone combinations__:
+  * __FPN__: Use a ResNet+FPN backbone with standard conv and FC heads for mask and box prediction,
+    respectively. It obtains the best
+    speed/accuracy tradeoff, but the other two are still useful for research.
+  * __C4__: Use a ResNet conv4 backbone with conv5 head. The original baseline in the Faster R-CNN paper.
+  * __DC5__ (Dilated-C5): Use a ResNet conv5 backbone with dilations in conv5, and standard conv and FC heads
+    for mask and box prediction, respectively.
+    This is used by the Deformable ConvNet paper.
+* Most models are trained with the 3x schedule (~37 COCO epochs).
+  Although 1x models are heavily under-trained, we provide some ResNet-50 models with the 1x (~12 COCO epochs)
+  training schedule for comparison when doing quick research iteration.
+
+#### ImageNet Pretrained Models
+
+It's common to initialize from backbone models pre-trained on ImageNet classification tasks. The following backbone models are available:
+
+* [R-50.pkl](https://dl.fbaipublicfiles.com/detectron2/ImageNetPretrained/MSRA/R-50.pkl): converted copy of [MSRA's original ResNet-50](https://github.com/KaimingHe/deep-residual-networks) model.
+* [R-101.pkl](https://dl.fbaipublicfiles.com/detectron2/ImageNetPretrained/MSRA/R-101.pkl): converted copy of [MSRA's original ResNet-101](https://github.com/KaimingHe/deep-residual-networks) model.
+* [X-101-32x8d.pkl](https://dl.fbaipublicfiles.com/detectron2/ImageNetPretrained/FAIR/X-101-32x8d.pkl): ResNeXt-101-32x8d model trained with Caffe2 at FB.
+* [R-50.pkl (torchvision)](https://dl.fbaipublicfiles.com/detectron2/ImageNetPretrained/torchvision/R-50.pkl): converted copy of [torchvision's ResNet-50](https://pytorch.org/docs/stable/torchvision/models.html#torchvision.models.resnet50) model.
+  More details can be found in [the conversion script](tools/convert-torchvision-to-d2.py).
+
+Note that the above models have __different__ format from those provided in Detectron: we do not fuse BatchNorm into an affine layer.
+Pretrained models in Detectron's format can still be used. For example:
+* [X-152-32x8d-IN5k.pkl](https://dl.fbaipublicfiles.com/detectron/ImageNetPretrained/25093814/X-152-32x8d-IN5k.pkl):
+  ResNeXt-152-32x8d model trained on ImageNet-5k with Caffe2 at FB (see ResNeXt paper for details on ImageNet-5k).
+* [R-50-GN.pkl](https://dl.fbaipublicfiles.com/detectron/ImageNetPretrained/47261647/R-50-GN.pkl):
+  ResNet-50 with Group Normalization.
+* [R-101-GN.pkl](https://dl.fbaipublicfiles.com/detectron/ImageNetPretrained/47592356/R-101-GN.pkl):
+  ResNet-101 with Group Normalization.
+
+These models require slightly different settings regarding normalization and architecture. See the model zoo configs for reference.
+
+#### License
+
+All models available for download through this document are licensed under the
+[Creative Commons Attribution-ShareAlike 3.0 license](https://creativecommons.org/licenses/by-sa/3.0/).
+
+### COCO Object Detection Baselines
+
+#### Faster R-CNN:
+<!--
+(fb only) To update the table in vim:
+1. Remove the old table: d}
+2. Copy the below command to the place of the table
+3. :.!bash
+
+./gen_html_table.py --config 'COCO-Detection/faster*50*'{1x,3x}'*' 'COCO-Detection/faster*101*' --name R50-C4 R50-DC5 R50-FPN R50-C4 R50-DC5 R50-FPN R101-C4 R101-DC5 R101-FPN X101-FPN --fields lr_sched train_speed inference_speed mem box_AP
+-->
+
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">lr<br/>sched</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: faster_rcnn_R_50_C4_1x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_R_50_C4_1x.yaml">R50-C4</a></td>
+<td align="center">1x</td>
+<td align="center">0.551</td>
+<td align="center">0.102</td>
+<td align="center">4.8</td>
+<td align="center">35.7</td>
+<td align="center">137257644</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_C4_1x/137257644/model_final_721ade.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_C4_1x/137257644/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_R_50_DC5_1x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_R_50_DC5_1x.yaml">R50-DC5</a></td>
+<td align="center">1x</td>
+<td align="center">0.380</td>
+<td align="center">0.068</td>
+<td align="center">5.0</td>
+<td align="center">37.3</td>
+<td align="center">137847829</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_DC5_1x/137847829/model_final_51d356.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_DC5_1x/137847829/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_R_50_FPN_1x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_R_50_FPN_1x.yaml">R50-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.210</td>
+<td align="center">0.038</td>
+<td align="center">3.0</td>
+<td align="center">37.9</td>
+<td align="center">137257794</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_FPN_1x/137257794/model_final_b275ba.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_FPN_1x/137257794/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_R_50_C4_3x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_R_50_C4_3x.yaml">R50-C4</a></td>
+<td align="center">3x</td>
+<td align="center">0.543</td>
+<td align="center">0.104</td>
+<td align="center">4.8</td>
+<td align="center">38.4</td>
+<td align="center">137849393</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_C4_3x/137849393/model_final_f97cb7.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_C4_3x/137849393/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_R_50_DC5_3x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_R_50_DC5_3x.yaml">R50-DC5</a></td>
+<td align="center">3x</td>
+<td align="center">0.378</td>
+<td align="center">0.070</td>
+<td align="center">5.0</td>
+<td align="center">39.0</td>
+<td align="center">137849425</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_DC5_3x/137849425/model_final_68d202.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_DC5_3x/137849425/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_R_50_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml">R50-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.209</td>
+<td align="center">0.038</td>
+<td align="center">3.0</td>
+<td align="center">40.2</td>
+<td align="center">137849458</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_FPN_3x/137849458/model_final_280758.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_50_FPN_3x/137849458/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_R_101_C4_3x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_R_101_C4_3x.yaml">R101-C4</a></td>
+<td align="center">3x</td>
+<td align="center">0.619</td>
+<td align="center">0.139</td>
+<td align="center">5.9</td>
+<td align="center">41.1</td>
+<td align="center">138204752</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_101_C4_3x/138204752/model_final_298dad.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_101_C4_3x/138204752/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_R_101_DC5_3x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_R_101_DC5_3x.yaml">R101-DC5</a></td>
+<td align="center">3x</td>
+<td align="center">0.452</td>
+<td align="center">0.086</td>
+<td align="center">6.1</td>
+<td align="center">40.6</td>
+<td align="center">138204841</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_101_DC5_3x/138204841/model_final_3e0943.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_101_DC5_3x/138204841/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_R_101_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_R_101_FPN_3x.yaml">R101-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.286</td>
+<td align="center">0.051</td>
+<td align="center">4.1</td>
+<td align="center">42.0</td>
+<td align="center">137851257</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_101_FPN_3x/137851257/model_final_f6e8b1.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_101_FPN_3x/137851257/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_X_101_32x8d_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x.yaml">X101-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.638</td>
+<td align="center">0.098</td>
+<td align="center">6.7</td>
+<td align="center">43.0</td>
+<td align="center">139173657</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x/139173657/model_final_68b088.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x/139173657/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+#### RetinaNet:
+<!--
+./gen_html_table.py --config 'COCO-Detection/retina*50*' 'COCO-Detection/retina*101*' --name R50 R50 R101 --fields lr_sched train_speed inference_speed mem box_AP
+-->
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">lr<br/>sched</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: retinanet_R_50_FPN_1x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/retinanet_R_50_FPN_1x.yaml">R50</a></td>
+<td align="center">1x</td>
+<td align="center">0.205</td>
+<td align="center">0.041</td>
+<td align="center">4.1</td>
+<td align="center">37.4</td>
+<td align="center">190397773</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/retinanet_R_50_FPN_1x/190397773/model_final_bfca0b.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/retinanet_R_50_FPN_1x/190397773/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: retinanet_R_50_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/retinanet_R_50_FPN_3x.yaml">R50</a></td>
+<td align="center">3x</td>
+<td align="center">0.205</td>
+<td align="center">0.041</td>
+<td align="center">4.1</td>
+<td align="center">38.7</td>
+<td align="center">190397829</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/retinanet_R_50_FPN_3x/190397829/model_final_5bd44e.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/retinanet_R_50_FPN_3x/190397829/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: retinanet_R_101_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/retinanet_R_101_FPN_3x.yaml">R101</a></td>
+<td align="center">3x</td>
+<td align="center">0.291</td>
+<td align="center">0.054</td>
+<td align="center">5.2</td>
+<td align="center">40.4</td>
+<td align="center">190397697</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/retinanet_R_101_FPN_3x/190397697/model_final_971ab9.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/retinanet_R_101_FPN_3x/190397697/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+
+#### RPN & Fast R-CNN:
+<!--
+./gen_html_table.py --config 'COCO-Detection/rpn*' 'COCO-Detection/fast_rcnn*' --name "RPN R50-C4" "RPN R50-FPN" "Fast R-CNN R50-FPN" --fields lr_sched train_speed inference_speed mem box_AP prop_AR
+-->
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">lr<br/>sched</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">prop.<br/>AR</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: rpn_R_50_C4_1x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/rpn_R_50_C4_1x.yaml">RPN R50-C4</a></td>
+<td align="center">1x</td>
+<td align="center">0.130</td>
+<td align="center">0.034</td>
+<td align="center">1.5</td>
+<td align="center"></td>
+<td align="center">51.6</td>
+<td align="center">137258005</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/rpn_R_50_C4_1x/137258005/model_final_450694.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/rpn_R_50_C4_1x/137258005/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: rpn_R_50_FPN_1x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/rpn_R_50_FPN_1x.yaml">RPN R50-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.186</td>
+<td align="center">0.032</td>
+<td align="center">2.7</td>
+<td align="center"></td>
+<td align="center">58.0</td>
+<td align="center">137258492</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/rpn_R_50_FPN_1x/137258492/model_final_02ce48.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/rpn_R_50_FPN_1x/137258492/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: fast_rcnn_R_50_FPN_1x -->
+ <tr><td align="left"><a href="configs/COCO-Detection/fast_rcnn_R_50_FPN_1x.yaml">Fast R-CNN R50-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.140</td>
+<td align="center">0.029</td>
+<td align="center">2.6</td>
+<td align="center">37.8</td>
+<td align="center"></td>
+<td align="center">137635226</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/fast_rcnn_R_50_FPN_1x/137635226/model_final_e5f7ce.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/fast_rcnn_R_50_FPN_1x/137635226/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+### COCO Instance Segmentation Baselines with Mask R-CNN
+<!--
+./gen_html_table.py --config 'COCO-InstanceSegmentation/mask*50*'{1x,3x}'*' 'COCO-InstanceSegmentation/mask*101*' --name R50-C4 R50-DC5 R50-FPN R50-C4 R50-DC5 R50-FPN R101-C4 R101-DC5 R101-FPN X101-FPN --fields lr_sched train_speed inference_speed mem box_AP mask_AP
+-->
+
+
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">lr<br/>sched</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">mask<br/>AP</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: mask_rcnn_R_50_C4_1x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x.yaml">R50-C4</a></td>
+<td align="center">1x</td>
+<td align="center">0.584</td>
+<td align="center">0.110</td>
+<td align="center">5.2</td>
+<td align="center">36.8</td>
+<td align="center">32.2</td>
+<td align="center">137259246</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x/137259246/model_final_9243eb.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x/137259246/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_DC5_1x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_1x.yaml">R50-DC5</a></td>
+<td align="center">1x</td>
+<td align="center">0.471</td>
+<td align="center">0.076</td>
+<td align="center">6.5</td>
+<td align="center">38.3</td>
+<td align="center">34.2</td>
+<td align="center">137260150</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_1x/137260150/model_final_4f86c3.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_1x/137260150/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_FPN_1x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml">R50-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.261</td>
+<td align="center">0.043</td>
+<td align="center">3.4</td>
+<td align="center">38.6</td>
+<td align="center">35.2</td>
+<td align="center">137260431</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x/137260431/model_final_a54504.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x/137260431/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_C4_3x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x.yaml">R50-C4</a></td>
+<td align="center">3x</td>
+<td align="center">0.575</td>
+<td align="center">0.111</td>
+<td align="center">5.2</td>
+<td align="center">39.8</td>
+<td align="center">34.4</td>
+<td align="center">137849525</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x/137849525/model_final_4ce675.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x/137849525/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_DC5_3x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x.yaml">R50-DC5</a></td>
+<td align="center">3x</td>
+<td align="center">0.470</td>
+<td align="center">0.076</td>
+<td align="center">6.5</td>
+<td align="center">40.0</td>
+<td align="center">35.9</td>
+<td align="center">137849551</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x/137849551/model_final_84107b.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x/137849551/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml">R50-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.261</td>
+<td align="center">0.043</td>
+<td align="center">3.4</td>
+<td align="center">41.0</td>
+<td align="center">37.2</td>
+<td align="center">137849600</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_101_C4_3x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_101_C4_3x.yaml">R101-C4</a></td>
+<td align="center">3x</td>
+<td align="center">0.652</td>
+<td align="center">0.145</td>
+<td align="center">6.3</td>
+<td align="center">42.6</td>
+<td align="center">36.7</td>
+<td align="center">138363239</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_101_C4_3x/138363239/model_final_a2914c.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_101_C4_3x/138363239/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_101_DC5_3x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_101_DC5_3x.yaml">R101-DC5</a></td>
+<td align="center">3x</td>
+<td align="center">0.545</td>
+<td align="center">0.092</td>
+<td align="center">7.6</td>
+<td align="center">41.9</td>
+<td align="center">37.3</td>
+<td align="center">138363294</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_101_DC5_3x/138363294/model_final_0464b7.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_101_DC5_3x/138363294/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_101_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x.yaml">R101-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.340</td>
+<td align="center">0.056</td>
+<td align="center">4.6</td>
+<td align="center">42.9</td>
+<td align="center">38.6</td>
+<td align="center">138205316</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x/138205316/model_final_a3ec72.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x/138205316/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_X_101_32x8d_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x.yaml">X101-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.690</td>
+<td align="center">0.103</td>
+<td align="center">7.2</td>
+<td align="center">44.3</td>
+<td align="center">39.5</td>
+<td align="center">139653917</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x/139653917/model_final_2d9806.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x/139653917/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+
+
+#### New baselines using Large-Scale Jitter and Longer Training Schedule
+
+The following baselines of COCO Instance Segmentation with Mask R-CNN are generated
+using a longer training schedule and large-scale jitter as described in Google's
+[Simple Copy-Paste Data Augmentation](https://arxiv.org/pdf/2012.07177.pdf) paper. These
+models are trained from scratch using random initialization. These baselines exceed the
+previous Mask R-CNN baselines.
+
+In the following table, one epoch consists of training on 118000 COCO images.
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">epochs</th>
+<th valign="bottom">train<br/>time<br/>(s/im)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">mask<br/>AP</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: mask_rcnn_R_50_FPN_100ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_R_50_FPN_100ep_LSJ.py">R50-FPN</a></td>
+<td align="center">100</td>
+<td align="center">0.376</td>
+<td align="center">0.069</td>
+<td align="center">44.6</td>
+<td align="center">40.3</td>
+<td align="center">42047764</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_50_FPN_100ep_LSJ/42047764/model_final_bb69de.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_50_FPN_100ep_LSJ/42047764/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_FPN_200ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_R_50_FPN_200ep_LSJ.py">R50-FPN</a></td>
+<td align="center">200</td>
+<td align="center">0.376</td>
+<td align="center">0.069</td>
+<td align="center">46.3</td>
+<td align="center">41.7</td>
+<td align="center">42047638</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_50_FPN_200ep_LSJ/42047638/model_final_89a8d3.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_50_FPN_200ep_LSJ/42047638/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_FPN_400ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_R_50_FPN_400ep_LSJ.py">R50-FPN</a></td>
+<td align="center">400</td>
+<td align="center">0.376</td>
+<td align="center">0.069</td>
+<td align="center">47.4</td>
+<td align="center">42.5</td>
+<td align="center">42019571</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_50_FPN_400ep_LSJ/42019571/model_final_14d201.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_50_FPN_400ep_LSJ/42019571/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_101_FPN_100ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_R_101_FPN_100ep_LSJ.py">R101-FPN</a></td>
+<td align="center">100</td>
+<td align="center">0.518</td>
+<td align="center">0.073</td>
+<td align="center">46.4</td>
+<td align="center">41.6</td>
+<td align="center">42025812</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_101_FPN_100ep_LSJ/42025812/model_final_4f7b58.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_101_FPN_100ep_LSJ/42025812/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_101_FPN_200ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_R_101_FPN_200ep_LSJ.py">R101-FPN</a></td>
+<td align="center">200</td>
+<td align="center">0.518</td>
+<td align="center">0.073</td>
+<td align="center">48.0</td>
+<td align="center">43.1</td>
+<td align="center">42131867</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_101_FPN_200ep_LSJ/42131867/model_final_0bb7ae.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_101_FPN_200ep_LSJ/42131867/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_101_FPN_400ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_R_101_FPN_400ep_LSJ.py">R101-FPN</a></td>
+<td align="center">400</td>
+<td align="center">0.518</td>
+<td align="center">0.073</td>
+<td align="center">48.9</td>
+<td align="center">43.7</td>
+<td align="center">42073830</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_101_FPN_400ep_LSJ/42073830/model_final_f96b26.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_R_101_FPN_400ep_LSJ/42073830/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_regnetx_4gf_dds_FPN_100ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_100ep_LSJ.py">regnetx_4gf_dds_FPN</a></td>
+<td align="center">100</td>
+<td align="center">0.474</td>
+<td align="center">0.071</td>
+<td align="center">46.0</td>
+<td align="center">41.3</td>
+<td align="center">42047771</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_100ep_LSJ/42047771/model_final_b7fbab.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_100ep_LSJ/42047771/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_regnetx_4gf_dds_FPN_200ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_200ep_LSJ.py">regnetx_4gf_dds_FPN</a></td>
+<td align="center">200</td>
+<td align="center">0.474</td>
+<td align="center">0.071</td>
+<td align="center">48.1</td>
+<td align="center">43.1</td>
+<td align="center">42132721</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_200ep_LSJ/42132721/model_final_5d87c1.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_200ep_LSJ/42132721/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_regnetx_4gf_dds_FPN_400ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_400ep_LSJ.py">regnetx_4gf_dds_FPN</a></td>
+<td align="center">400</td>
+<td align="center">0.474</td>
+<td align="center">0.071</td>
+<td align="center">48.6</td>
+<td align="center">43.5</td>
+<td align="center">42025447</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_400ep_LSJ/42025447/model_final_f1362d.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_400ep_LSJ/42025447/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_regnety_4gf_dds_FPN_100ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_regnety_4gf_dds_FPN_100ep_LSJ.py">regnety_4gf_dds_FPN</a></td>
+<td align="center">100</td>
+<td align="center">0.487</td>
+<td align="center">0.073</td>
+<td align="center">46.1</td>
+<td align="center">41.6</td>
+<td align="center">42047784</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnety_4gf_dds_FPN_100ep_LSJ/42047784/model_final_6ba57e.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnety_4gf_dds_FPN_100ep_LSJ/42047784/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_regnety_4gf_dds_FPN_200ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_regnety_4gf_dds_FPN_200ep_LSJ.py">regnety_4gf_dds_FPN</a></td>
+<td align="center">200</td>
+<td align="center">0.487</td>
+<td align="center">0.072</td>
+<td align="center">47.8</td>
+<td align="center">43.0</td>
+<td align="center">42047642</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnety_4gf_dds_FPN_200ep_LSJ/42047642/model_final_27b9c1.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnety_4gf_dds_FPN_200ep_LSJ/42047642/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ -->
+ <tr><td align="left"><a href="configs/new_baselines/mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ.py">regnety_4gf_dds_FPN</a></td>
+<td align="center">400</td>
+<td align="center">0.487</td>
+<td align="center">0.072</td>
+<td align="center">48.2</td>
+<td align="center">43.3</td>
+<td align="center">42045954</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ/42045954/model_final_ef3a80.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/new_baselines/mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ/42045954/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+### COCO Person Keypoint Detection Baselines with Keypoint R-CNN
+<!--
+./gen_html_table.py --config 'COCO-Keypoints/*50*' 'COCO-Keypoints/*101*'  --name R50-FPN R50-FPN R101-FPN X101-FPN --fields lr_sched train_speed inference_speed mem box_AP keypoint_AP
+-->
+
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">lr<br/>sched</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">kp.<br/>AP</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: keypoint_rcnn_R_50_FPN_1x -->
+ <tr><td align="left"><a href="configs/COCO-Keypoints/keypoint_rcnn_R_50_FPN_1x.yaml">R50-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.315</td>
+<td align="center">0.072</td>
+<td align="center">5.0</td>
+<td align="center">53.6</td>
+<td align="center">64.0</td>
+<td align="center">137261548</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Keypoints/keypoint_rcnn_R_50_FPN_1x/137261548/model_final_04e291.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Keypoints/keypoint_rcnn_R_50_FPN_1x/137261548/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: keypoint_rcnn_R_50_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml">R50-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.316</td>
+<td align="center">0.066</td>
+<td align="center">5.0</td>
+<td align="center">55.4</td>
+<td align="center">65.5</td>
+<td align="center">137849621</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x/137849621/model_final_a6e10b.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x/137849621/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: keypoint_rcnn_R_101_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-Keypoints/keypoint_rcnn_R_101_FPN_3x.yaml">R101-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.390</td>
+<td align="center">0.076</td>
+<td align="center">6.1</td>
+<td align="center">56.4</td>
+<td align="center">66.1</td>
+<td align="center">138363331</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Keypoints/keypoint_rcnn_R_101_FPN_3x/138363331/model_final_997cc7.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Keypoints/keypoint_rcnn_R_101_FPN_3x/138363331/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: keypoint_rcnn_X_101_32x8d_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x.yaml">X101-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.738</td>
+<td align="center">0.121</td>
+<td align="center">8.7</td>
+<td align="center">57.3</td>
+<td align="center">66.0</td>
+<td align="center">139686956</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x/139686956/model_final_5ad38f.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x/139686956/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+### COCO Panoptic Segmentation Baselines with Panoptic FPN
+<!--
+./gen_html_table.py --config 'COCO-PanopticSegmentation/*50*' 'COCO-PanopticSegmentation/*101*'  --name R50-FPN R50-FPN R101-FPN --fields lr_sched train_speed inference_speed mem box_AP mask_AP PQ
+-->
+
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">lr<br/>sched</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">mask<br/>AP</th>
+<th valign="bottom">PQ</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: panoptic_fpn_R_50_1x -->
+ <tr><td align="left"><a href="configs/COCO-PanopticSegmentation/panoptic_fpn_R_50_1x.yaml">R50-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.304</td>
+<td align="center">0.053</td>
+<td align="center">4.8</td>
+<td align="center">37.6</td>
+<td align="center">34.7</td>
+<td align="center">39.4</td>
+<td align="center">139514544</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-PanopticSegmentation/panoptic_fpn_R_50_1x/139514544/model_final_dbfeb4.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-PanopticSegmentation/panoptic_fpn_R_50_1x/139514544/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: panoptic_fpn_R_50_3x -->
+ <tr><td align="left"><a href="configs/COCO-PanopticSegmentation/panoptic_fpn_R_50_3x.yaml">R50-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.302</td>
+<td align="center">0.053</td>
+<td align="center">4.8</td>
+<td align="center">40.0</td>
+<td align="center">36.5</td>
+<td align="center">41.5</td>
+<td align="center">139514569</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-PanopticSegmentation/panoptic_fpn_R_50_3x/139514569/model_final_c10459.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-PanopticSegmentation/panoptic_fpn_R_50_3x/139514569/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: panoptic_fpn_R_101_3x -->
+ <tr><td align="left"><a href="configs/COCO-PanopticSegmentation/panoptic_fpn_R_101_3x.yaml">R101-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.392</td>
+<td align="center">0.066</td>
+<td align="center">6.0</td>
+<td align="center">42.4</td>
+<td align="center">38.5</td>
+<td align="center">43.0</td>
+<td align="center">139514519</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-PanopticSegmentation/panoptic_fpn_R_101_3x/139514519/model_final_cafdb1.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-PanopticSegmentation/panoptic_fpn_R_101_3x/139514519/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+
+### LVIS Instance Segmentation Baselines with Mask R-CNN
+
+Mask R-CNN baselines on the [LVIS dataset](https://lvisdataset.org), v0.5.
+These baselines are described in Table 3(c) of the [LVIS paper](https://arxiv.org/abs/1908.03195).
+
+NOTE: the 1x schedule here has the same amount of __iterations__ as the COCO 1x baselines.
+They are roughly 24 epochs of LVISv0.5 data.
+The final results of these configs have large variance across different runs.
+
+<!--
+./gen_html_table.py --config 'LVISv0.5-InstanceSegmentation/mask*50*' 'LVISv0.5-InstanceSegmentation/mask*101*' --name R50-FPN R101-FPN X101-FPN --fields lr_sched train_speed inference_speed mem box_AP mask_AP
+-->
+
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">lr<br/>sched</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">mask<br/>AP</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: mask_rcnn_R_50_FPN_1x -->
+ <tr><td align="left"><a href="configs/LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml">R50-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.292</td>
+<td align="center">0.107</td>
+<td align="center">7.1</td>
+<td align="center">23.6</td>
+<td align="center">24.4</td>
+<td align="center">144219072</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x/144219072/model_final_571f7c.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x/144219072/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_101_FPN_1x -->
+ <tr><td align="left"><a href="configs/LVISv0.5-InstanceSegmentation/mask_rcnn_R_101_FPN_1x.yaml">R101-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.371</td>
+<td align="center">0.114</td>
+<td align="center">7.8</td>
+<td align="center">25.6</td>
+<td align="center">25.9</td>
+<td align="center">144219035</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/LVISv0.5-InstanceSegmentation/mask_rcnn_R_101_FPN_1x/144219035/model_final_824ab5.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/LVISv0.5-InstanceSegmentation/mask_rcnn_R_101_FPN_1x/144219035/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_X_101_32x8d_FPN_1x -->
+ <tr><td align="left"><a href="configs/LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x.yaml">X101-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.712</td>
+<td align="center">0.151</td>
+<td align="center">10.2</td>
+<td align="center">26.7</td>
+<td align="center">27.1</td>
+<td align="center">144219108</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x/144219108/model_final_5e3439.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x/144219108/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+
+
+### Cityscapes & Pascal VOC Baselines
+
+Simple baselines for
+* Mask R-CNN on Cityscapes instance segmentation (initialized from COCO pre-training, then trained on Cityscapes fine annotations only)
+* Faster R-CNN on PASCAL VOC object detection (trained on VOC 2007 train+val + VOC 2012 train+val, tested on VOC 2007 using 11-point interpolated AP)
+
+<!--
+./gen_html_table.py --config 'Cityscapes/*' 'PascalVOC-Detection/*' --name "R50-FPN, Cityscapes" "R50-C4, VOC" --fields train_speed inference_speed mem box_AP box_AP50 mask_AP
+-->
+
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">box<br/>AP50</th>
+<th valign="bottom">mask<br/>AP</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: mask_rcnn_R_50_FPN -->
+ <tr><td align="left"><a href="configs/Cityscapes/mask_rcnn_R_50_FPN.yaml">R50-FPN, Cityscapes</a></td>
+<td align="center">0.240</td>
+<td align="center">0.078</td>
+<td align="center">4.4</td>
+<td align="center"></td>
+<td align="center"></td>
+<td align="center">36.5</td>
+<td align="center">142423278</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Cityscapes/mask_rcnn_R_50_FPN/142423278/model_final_af9cf5.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Cityscapes/mask_rcnn_R_50_FPN/142423278/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: faster_rcnn_R_50_C4 -->
+ <tr><td align="left"><a href="configs/PascalVOC-Detection/faster_rcnn_R_50_C4.yaml">R50-C4, VOC</a></td>
+<td align="center">0.537</td>
+<td align="center">0.081</td>
+<td align="center">4.8</td>
+<td align="center">51.9</td>
+<td align="center">80.3</td>
+<td align="center"></td>
+<td align="center">142202221</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/PascalVOC-Detection/faster_rcnn_R_50_C4/142202221/model_final_b1acc2.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/PascalVOC-Detection/faster_rcnn_R_50_C4/142202221/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+
+
+### Other Settings
+
+Ablations for Deformable Conv and Cascade R-CNN:
+
+<!--
+./gen_html_table.py --config 'COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml' 'Misc/*R_50_FPN_1x_dconv*' 'Misc/cascade*1x.yaml' 'COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml' 'Misc/*R_50_FPN_3x_dconv*' 'Misc/cascade*3x.yaml' --name "Baseline R50-FPN" "Deformable Conv" "Cascade R-CNN" "Baseline R50-FPN" "Deformable Conv" "Cascade R-CNN"  --fields lr_sched train_speed inference_speed mem box_AP mask_AP
+-->
+
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">lr<br/>sched</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">mask<br/>AP</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: mask_rcnn_R_50_FPN_1x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml">Baseline R50-FPN</a></td>
+<td align="center">1x</td>
+<td align="center">0.261</td>
+<td align="center">0.043</td>
+<td align="center">3.4</td>
+<td align="center">38.6</td>
+<td align="center">35.2</td>
+<td align="center">137260431</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x/137260431/model_final_a54504.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x/137260431/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_FPN_1x_dconv_c3-c5 -->
+ <tr><td align="left"><a href="configs/Misc/mask_rcnn_R_50_FPN_1x_dconv_c3-c5.yaml">Deformable Conv</a></td>
+<td align="center">1x</td>
+<td align="center">0.342</td>
+<td align="center">0.048</td>
+<td align="center">3.5</td>
+<td align="center">41.5</td>
+<td align="center">37.5</td>
+<td align="center">138602867</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/mask_rcnn_R_50_FPN_1x_dconv_c3-c5/138602867/model_final_65c703.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/mask_rcnn_R_50_FPN_1x_dconv_c3-c5/138602867/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: cascade_mask_rcnn_R_50_FPN_1x -->
+ <tr><td align="left"><a href="configs/Misc/cascade_mask_rcnn_R_50_FPN_1x.yaml">Cascade R-CNN</a></td>
+<td align="center">1x</td>
+<td align="center">0.317</td>
+<td align="center">0.052</td>
+<td align="center">4.0</td>
+<td align="center">42.1</td>
+<td align="center">36.4</td>
+<td align="center">138602847</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/cascade_mask_rcnn_R_50_FPN_1x/138602847/model_final_e9d89b.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/cascade_mask_rcnn_R_50_FPN_1x/138602847/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml">Baseline R50-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.261</td>
+<td align="center">0.043</td>
+<td align="center">3.4</td>
+<td align="center">41.0</td>
+<td align="center">37.2</td>
+<td align="center">137849600</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_FPN_3x_dconv_c3-c5 -->
+ <tr><td align="left"><a href="configs/Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5.yaml">Deformable Conv</a></td>
+<td align="center">3x</td>
+<td align="center">0.349</td>
+<td align="center">0.047</td>
+<td align="center">3.5</td>
+<td align="center">42.7</td>
+<td align="center">38.5</td>
+<td align="center">144998336</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5/144998336/model_final_821d0b.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5/144998336/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: cascade_mask_rcnn_R_50_FPN_3x -->
+ <tr><td align="left"><a href="configs/Misc/cascade_mask_rcnn_R_50_FPN_3x.yaml">Cascade R-CNN</a></td>
+<td align="center">3x</td>
+<td align="center">0.328</td>
+<td align="center">0.053</td>
+<td align="center">4.0</td>
+<td align="center">44.3</td>
+<td align="center">38.5</td>
+<td align="center">144998488</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/cascade_mask_rcnn_R_50_FPN_3x/144998488/model_final_480dd8.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/cascade_mask_rcnn_R_50_FPN_3x/144998488/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+
+Ablations for normalization methods, and a few models trained from scratch following [Rethinking ImageNet Pre-training](https://arxiv.org/abs/1811.08883).
+(Note: The baseline uses `2fc` head while the others use [`4conv1fc` head](https://arxiv.org/abs/1803.08494))
+<!--
+./gen_html_table.py --config 'COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml' 'Misc/mask*50_FPN_3x_gn.yaml' 'Misc/mask*50_FPN_3x_syncbn.yaml' 'Misc/scratch*' --name "Baseline R50-FPN" "GN" "SyncBN" "GN (from scratch)" "GN (from scratch)" "SyncBN (from scratch)" --fields lr_sched train_speed inference_speed mem box_AP mask_AP
+   -->
+
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">lr<br/>sched</th>
+<th valign="bottom">train<br/>time<br/>(s/iter)</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">mask<br/>AP</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: mask_rcnn_R_50_FPN_3x -->
+ <tr><td align="left"><a href="configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml">Baseline R50-FPN</a></td>
+<td align="center">3x</td>
+<td align="center">0.261</td>
+<td align="center">0.043</td>
+<td align="center">3.4</td>
+<td align="center">41.0</td>
+<td align="center">37.2</td>
+<td align="center">137849600</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_FPN_3x_gn -->
+ <tr><td align="left"><a href="configs/Misc/mask_rcnn_R_50_FPN_3x_gn.yaml">GN</a></td>
+<td align="center">3x</td>
+<td align="center">0.309</td>
+<td align="center">0.060</td>
+<td align="center">5.6</td>
+<td align="center">42.6</td>
+<td align="center">38.6</td>
+<td align="center">138602888</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/mask_rcnn_R_50_FPN_3x_gn/138602888/model_final_dc5d9e.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/mask_rcnn_R_50_FPN_3x_gn/138602888/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: mask_rcnn_R_50_FPN_3x_syncbn -->
+ <tr><td align="left"><a href="configs/Misc/mask_rcnn_R_50_FPN_3x_syncbn.yaml">SyncBN</a></td>
+<td align="center">3x</td>
+<td align="center">0.345</td>
+<td align="center">0.053</td>
+<td align="center">5.5</td>
+<td align="center">41.9</td>
+<td align="center">37.8</td>
+<td align="center">169527823</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/mask_rcnn_R_50_FPN_3x_syncbn/169527823/model_final_3b3c51.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/mask_rcnn_R_50_FPN_3x_syncbn/169527823/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: scratch_mask_rcnn_R_50_FPN_3x_gn -->
+ <tr><td align="left"><a href="configs/Misc/scratch_mask_rcnn_R_50_FPN_3x_gn.yaml">GN (from scratch)</a></td>
+<td align="center">3x</td>
+<td align="center">0.338</td>
+<td align="center">0.061</td>
+<td align="center">7.2</td>
+<td align="center">39.9</td>
+<td align="center">36.6</td>
+<td align="center">138602908</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/scratch_mask_rcnn_R_50_FPN_3x_gn/138602908/model_final_01ca85.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/scratch_mask_rcnn_R_50_FPN_3x_gn/138602908/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: scratch_mask_rcnn_R_50_FPN_9x_gn -->
+ <tr><td align="left"><a href="configs/Misc/scratch_mask_rcnn_R_50_FPN_9x_gn.yaml">GN (from scratch)</a></td>
+<td align="center">9x</td>
+<td align="center">N/A</td>
+<td align="center">0.061</td>
+<td align="center">7.2</td>
+<td align="center">43.7</td>
+<td align="center">39.6</td>
+<td align="center">183808979</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/scratch_mask_rcnn_R_50_FPN_9x_gn/183808979/model_final_da7b4c.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/scratch_mask_rcnn_R_50_FPN_9x_gn/183808979/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: scratch_mask_rcnn_R_50_FPN_9x_syncbn -->
+ <tr><td align="left"><a href="configs/Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn.yaml">SyncBN (from scratch)</a></td>
+<td align="center">9x</td>
+<td align="center">N/A</td>
+<td align="center">0.055</td>
+<td align="center">7.2</td>
+<td align="center">43.6</td>
+<td align="center">39.3</td>
+<td align="center">184226666</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn/184226666/model_final_5ce33e.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn/184226666/metrics.json">metrics</a></td>
+</tr>
+</tbody></table>
+
+
+A few very large models trained for a long time, for demo purposes. They are trained using multiple machines:
+
+<!--
+./gen_html_table.py --config 'Misc/panoptic_*dconv*' 'Misc/cascade_*152*' --name "Panoptic FPN R101" "Mask R-CNN X152" --fields inference_speed mem box_AP mask_AP PQ
+# manually add TTA results
+-->
+
+
+<table><tbody>
+<!-- START TABLE -->
+<!-- TABLE HEADER -->
+<th valign="bottom">Name</th>
+<th valign="bottom">inference<br/>time<br/>(s/im)</th>
+<th valign="bottom">train<br/>mem<br/>(GB)</th>
+<th valign="bottom">box<br/>AP</th>
+<th valign="bottom">mask<br/>AP</th>
+<th valign="bottom">PQ</th>
+<th valign="bottom">model id</th>
+<th valign="bottom">download</th>
+<!-- TABLE BODY -->
+<!-- ROW: panoptic_fpn_R_101_dconv_cascade_gn_3x -->
+ <tr><td align="left"><a href="configs/Misc/panoptic_fpn_R_101_dconv_cascade_gn_3x.yaml">Panoptic FPN R101</a></td>
+<td align="center">0.098</td>
+<td align="center">11.4</td>
+<td align="center">47.4</td>
+<td align="center">41.3</td>
+<td align="center">46.1</td>
+<td align="center">139797668</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/panoptic_fpn_R_101_dconv_cascade_gn_3x/139797668/model_final_be35db.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/panoptic_fpn_R_101_dconv_cascade_gn_3x/139797668/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv -->
+ <tr><td align="left"><a href="configs/Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv.yaml">Mask R-CNN X152</a></td>
+<td align="center">0.234</td>
+<td align="center">15.1</td>
+<td align="center">50.2</td>
+<td align="center">44.0</td>
+<td align="center"></td>
+<td align="center">18131413</td>
+<td align="center"><a href="https://dl.fbaipublicfiles.com/detectron2/Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv/18131413/model_0039999_e76410.pkl">model</a>&nbsp;|&nbsp;<a href="https://dl.fbaipublicfiles.com/detectron2/Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv/18131413/metrics.json">metrics</a></td>
+</tr>
+<!-- ROW: TTA cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv -->
+ <tr><td align="left">above + test-time aug.</td>
+<td align="center"></td>
+<td align="center"></td>
+<td align="center">51.9</td>
+<td align="center">45.9</td>
+<td align="center"></td>
+<td align="center"></td>
+<td align="center"></td>
+</tr>
+</tbody></table>

approach/ovod/detectron2/README.md

@@ -0,0 +1,68 @@
+<img src=".github/Detectron2-Logo-Horz.svg" width="300" >
+
+<a href="https://opensource.facebook.com/support-ukraine">
+  <img src="https://img.shields.io/badge/Support-Ukraine-FFD500?style=flat&labelColor=005BBB" alt="Support Ukraine - Help Provide Humanitarian Aid to Ukraine." />
+</a>
+
+Detectron2 is Facebook AI Research's next generation library
+that provides state-of-the-art detection and segmentation algorithms.
+It is the successor of
+[Detectron](https://github.com/facebookresearch/Detectron/)
+and [maskrcnn-benchmark](https://github.com/facebookresearch/maskrcnn-benchmark/).
+It supports a number of computer vision research projects and production applications in Facebook.
+
+<div align="center">
+  <img src="https://user-images.githubusercontent.com/1381301/66535560-d3422200-eace-11e9-9123-5535d469db19.png"/>
+</div>
+<br>
+
+## Learn More about Detectron2
+
+Explain Like I’m 5: Detectron2            |  Using Machine Learning with Detectron2
+:-------------------------:|:-------------------------:
+[![Explain Like I’m 5: Detectron2](https://img.youtube.com/vi/1oq1Ye7dFqc/0.jpg)](https://www.youtube.com/watch?v=1oq1Ye7dFqc)  |  [![Using Machine Learning with Detectron2](https://img.youtube.com/vi/eUSgtfK4ivk/0.jpg)](https://www.youtube.com/watch?v=eUSgtfK4ivk)
+
+## What's New
+* Includes new capabilities such as panoptic segmentation, Densepose, Cascade R-CNN, rotated bounding boxes, PointRend,
+  DeepLab, ViTDet, MViTv2 etc.
+* Used as a library to support building [research projects](projects/) on top of it.
+* Models can be exported to TorchScript format or Caffe2 format for deployment.
+* It [trains much faster](https://detectron2.readthedocs.io/notes/benchmarks.html).
+
+See our [blog post](https://ai.facebook.com/blog/-detectron2-a-pytorch-based-modular-object-detection-library-/)
+to see more demos and learn about detectron2.
+
+## Installation
+
+See [installation instructions](https://detectron2.readthedocs.io/tutorials/install.html).
+
+## Getting Started
+
+See [Getting Started with Detectron2](https://detectron2.readthedocs.io/tutorials/getting_started.html),
+and the [Colab Notebook](https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5)
+to learn about basic usage.
+
+Learn more at our [documentation](https://detectron2.readthedocs.org).
+And see [projects/](projects/) for some projects that are built on top of detectron2.
+
+## Model Zoo and Baselines
+
+We provide a large set of baseline results and trained models available for download in the [Detectron2 Model Zoo](MODEL_ZOO.md).
+
+## License
+
+Detectron2 is released under the [Apache 2.0 license](LICENSE).
+
+## Citing Detectron2
+
+If you use Detectron2 in your research or wish to refer to the baseline results published in the [Model Zoo](MODEL_ZOO.md), please use the following BibTeX entry.
+
+```BibTeX
+@misc{wu2019detectron2,
+  author =       {Yuxin Wu and Alexander Kirillov and Francisco Massa and
+                  Wan-Yen Lo and Ross Girshick},
+  title =        {Detectron2},
+  howpublished = {\url{https://github.com/facebookresearch/detectron2}},
+  year =         {2019}
+}
+```

approach/ovod/detectron2/setup.cfg

@@ -0,0 +1,26 @@
+[isort]
+line_length=100
+multi_line_output=3
+include_trailing_comma=True
+known_standard_library=numpy,setuptools,mock
+skip=./datasets,docs
+skip_glob=*/__init__.py,**/configs/**,**/tests/config/**
+known_myself=detectron2
+known_third_party=fvcore,matplotlib,cv2,torch,torchvision,PIL,pycocotools,yacs,termcolor,cityscapesscripts,tabulate,tqdm,scipy,lvis,psutil,pkg_resources,caffe2,onnx,panopticapi,black,isort,av,iopath,omegaconf,hydra,yaml,pydoc,submitit,cloudpickle,packaging
+no_lines_before=STDLIB,THIRDPARTY
+sections=FUTURE,STDLIB,THIRDPARTY,myself,FIRSTPARTY,LOCALFOLDER
+default_section=FIRSTPARTY
+
+[mypy]
+python_version=3.7
+ignore_missing_imports = True
+warn_unused_configs = True
+disallow_untyped_defs = True
+check_untyped_defs = True
+warn_unused_ignores = True
+warn_redundant_casts = True
+show_column_numbers = True
+follow_imports = silent
+allow_redefinition = True
+; Require all functions to be annotated
+disallow_incomplete_defs = True

approach/ovod/detectron2/setup.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+import glob
+import os
+import shutil
+from os import path
+from setuptools import find_packages, setup
+from typing import List
+import torch
+from torch.utils.cpp_extension import CUDA_HOME, CppExtension, CUDAExtension
+
+torch_ver = [int(x) for x in torch.__version__.split(".")[:2]]
+assert torch_ver >= [1, 8], "Requires PyTorch >= 1.8"
+
+
+def get_version():
+    init_py_path = path.join(path.abspath(path.dirname(__file__)), "detectron2", "__init__.py")
+    init_py = open(init_py_path, "r").readlines()
+    version_line = [l.strip() for l in init_py if l.startswith("__version__")][0]
+    version = version_line.split("=")[-1].strip().strip("'\"")
+
+    # The following is used to build release packages.
+    # Users should never use it.
+    suffix = os.getenv("D2_VERSION_SUFFIX", "")
+    version = version + suffix
+    if os.getenv("BUILD_NIGHTLY", "0") == "1":
+        from datetime import datetime
+
+        date_str = datetime.today().strftime("%y%m%d")
+        version = version + ".dev" + date_str
+
+        new_init_py = [l for l in init_py if not l.startswith("__version__")]
+        new_init_py.append('__version__ = "{}"\n'.format(version))
+        with open(init_py_path, "w") as f:
+            f.write("".join(new_init_py))
+    return version
+
+
+def get_extensions():
+    this_dir = path.dirname(path.abspath(__file__))
+    extensions_dir = path.join(this_dir, "detectron2", "layers", "csrc")
+
+    main_source = path.join(extensions_dir, "vision.cpp")
+    sources = glob.glob(path.join(extensions_dir, "**", "*.cpp"))
+
+    from torch.utils.cpp_extension import ROCM_HOME
+
+    is_rocm_pytorch = (
+        True if ((torch.version.hip is not None) and (ROCM_HOME is not None)) else False
+    )
+    if is_rocm_pytorch:
+        assert torch_ver >= [1, 8], "ROCM support requires PyTorch >= 1.8!"
+
+    # common code between cuda and rocm platforms, for hipify version [1,0,0] and later.
+    source_cuda = glob.glob(path.join(extensions_dir, "**", "*.cu")) + glob.glob(
+        path.join(extensions_dir, "*.cu")
+    )
+    sources = [main_source] + sources
+
+    extension = CppExtension
+
+    extra_compile_args = {"cxx": []}
+    define_macros = []
+
+    if (torch.cuda.is_available() and ((CUDA_HOME is not None) or is_rocm_pytorch)) or os.getenv(
+        "FORCE_CUDA", "0"
+    ) == "1":
+        extension = CUDAExtension
+        sources += source_cuda
+
+        if not is_rocm_pytorch:
+            define_macros += [("WITH_CUDA", None)]
+            extra_compile_args["nvcc"] = [
+                "-O3",
+                "-DCUDA_HAS_FP16=1",
+                "-D__CUDA_NO_HALF_OPERATORS__",
+                "-D__CUDA_NO_HALF_CONVERSIONS__",
+                "-D__CUDA_NO_HALF2_OPERATORS__",
+            ]
+        else:
+            define_macros += [("WITH_HIP", None)]
+            extra_compile_args["nvcc"] = []
+
+        if torch_ver < [1, 7]:
+            # supported by https://github.com/pytorch/pytorch/pull/43931
+            CC = os.environ.get("CC", None)
+            if CC is not None:
+                extra_compile_args["nvcc"].append("-ccbin={}".format(CC))
+
+    include_dirs = [extensions_dir]
+
+    ext_modules = [
+        extension(
+            "detectron2._C",
+            sources,
+            include_dirs=include_dirs,
+            define_macros=define_macros,
+            extra_compile_args=extra_compile_args,
+        )
+    ]
+
+    return ext_modules
+
+
+def get_model_zoo_configs() -> List[str]:
+    """
+    Return a list of configs to include in package for model zoo. Copy over these configs inside
+    detectron2/model_zoo.
+    """
+
+    # Use absolute paths while symlinking.
+    source_configs_dir = path.join(path.dirname(path.realpath(__file__)), "configs")
+    destination = path.join(
+        path.dirname(path.realpath(__file__)), "detectron2", "model_zoo", "configs"
+    )
+    # Symlink the config directory inside package to have a cleaner pip install.
+
+    # Remove stale symlink/directory from a previous build.
+    if path.exists(source_configs_dir):
+        if path.islink(destination):
+            os.unlink(destination)
+        elif path.isdir(destination):
+            shutil.rmtree(destination)
+
+    if not path.exists(destination):
+        try:
+            os.symlink(source_configs_dir, destination)
+        except OSError:
+            # Fall back to copying if symlink fails: ex. on Windows.
+            shutil.copytree(source_configs_dir, destination)
+
+    config_paths = glob.glob("configs/**/*.yaml", recursive=True) + glob.glob(
+        "configs/**/*.py", recursive=True
+    )
+    return config_paths
+
+
+# For projects that are relative small and provide features that are very close
+# to detectron2's core functionalities, we install them under detectron2.projects
+PROJECTS = {
+    "detectron2.projects.point_rend": "projects/PointRend/point_rend",
+    "detectron2.projects.deeplab": "projects/DeepLab/deeplab",
+    "detectron2.projects.panoptic_deeplab": "projects/Panoptic-DeepLab/panoptic_deeplab",
+}
+
+setup(
+    name="detectron2",
+    version=get_version(),
+    author="FAIR",
+    url="https://github.com/facebookresearch/detectron2",
+    description="Detectron2 is FAIR's next-generation research "
+    "platform for object detection and segmentation.",
+    packages=find_packages(exclude=("configs", "tests*")) + list(PROJECTS.keys()),
+    package_dir=PROJECTS,
+    package_data={"detectron2.model_zoo": get_model_zoo_configs()},
+    python_requires=">=3.7",
+    install_requires=[
+        # These dependencies are not pure-python.
+        # In general, avoid adding more dependencies like them because they are not
+        # guaranteed to be installable by `pip install` on all platforms.
+        # To tell if a package is pure-python, go to https://pypi.org/project/{name}/#files
+        "Pillow>=7.1",  # or use pillow-simd for better performance
+        "matplotlib",  # TODO move it to optional after we add opencv visualization
+        "pycocotools>=2.0.2",  # corresponds to https://github.com/ppwwyyxx/cocoapi
+        # Do not add opencv here. Just like pytorch, user should install
+        # opencv themselves, preferrably by OS's package manager, or by
+        # choosing the proper pypi package name at https://github.com/skvark/opencv-python
+        # The following are pure-python dependencies that should be easily installable
+        "termcolor>=1.1",
+        "yacs>=0.1.8",
+        "tabulate",
+        "cloudpickle",
+        "tqdm>4.29.0",
+        "tensorboard",
+        # Lock version of fvcore/iopath because they may have breaking changes
+        # NOTE: when updating fvcore/iopath version, make sure fvcore depends
+        # on compatible version of iopath.
+        "fvcore>=0.1.5,<0.1.6",  # required like this to make it pip installable
+        "iopath>=0.1.7,<0.1.10",
+        "future",  # used by caffe2
+        "pydot",  # used to save caffe2 SVGs
+        "dataclasses; python_version<'3.7'",
+        "omegaconf>=2.1",
+        "hydra-core>=1.1",
+        "black==22.3.0",
+        "timm",
+        "fairscale",
+        "packaging",
+        # If a new dependency is required at import time (in addition to runtime), it
+        # probably needs to exist in docs/requirements.txt, or as a mock in docs/conf.py
+    ],
+    extras_require={
+        # optional dependencies, required by some features
+        "all": [
+            "scipy>1.5.1",
+            "shapely",
+            "pygments>=2.2",
+            "psutil",
+            "panopticapi @ https://github.com/cocodataset/panopticapi/archive/master.zip",
+        ],
+        # dev dependencies. Install them by `pip install 'detectron2[dev]'`
+        "dev": [
+            "flake8==3.8.1",
+            "isort==4.3.21",
+            "flake8-bugbear",
+            "flake8-comprehensions",
+        ],
+    },
+    ext_modules=get_extensions(),
+    cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension},
+)

approach/ovod/gdino.py

@@ -0,0 +1,23 @@
+from groundingdino.util.inference import load_model, load_image, predict, annotate
+import cv2
+
+model = load_model("./groundingdino/config/GroundingDINO_SwinT_OGC.py", "weights/groundingdino_swint_ogc.pth")
+
+# def peform_ovod(image_path, text_prompt, box_threshold, text_threshold)
+image_path = "weights/dog.jpg"
+text_prompt = "chair . person . dog ."
+box_threshold = 0.35
+text_threshold = 0.25
+
+image_source, image = load_image(image_path)
+
+boxes, logits, phrases = predict(
+    model=model,
+    image=image,
+    caption=text_prompt,
+    box_threshold=box_threshold,
+    text_threshold=text_threshold
+)
+
+annotated_frame = annotate(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)
+cv2.imwrite("annotated_image.jpg", annotated_frame)

approach/pipeline_utils.py

@@ -0,0 +1,176 @@
+import json
+import os
+from pathlib import Path
+import re
+import tempfile
+from typing import Iterable, List, Optional, Sequence
+
+
+IMAGE_NAME_PATTERN = re.compile(r"^(?P<app_id>[0-9]+)_(?P<frame_id>[0-9]+)\.[A-Za-z0-9]+$")
+
+
+def parse_orienter_image_name(image_name: str) -> tuple[str, str, int]:
+    path = Path(image_name)
+    if path.name != image_name:
+        raise ValueError(f"Image name must be a basename, not a path: {image_name}")
+    match = IMAGE_NAME_PATTERN.match(image_name)
+    if not match:
+        raise ValueError(
+            "Image name must match <app_id>_<frame>.<ext> with numeric app_id and frame: "
+            f"{image_name}"
+        )
+    app_id = match.group("app_id")
+    frame_id = match.group("frame_id")
+    frame_number = int(frame_id)
+    if frame_number > 999:
+        raise ValueError(f"Image frame must be in [0, 999] for the documented image_id: {image_name}")
+    return app_id, frame_id, int(f"{app_id}{frame_number:03d}")
+
+
+def resolve_image_path(images_dir: Path, image_name: str) -> Path:
+    """Resolve a manifest image path without allowing escape from images_dir."""
+    if not isinstance(image_name, str) or not image_name:
+        raise ValueError("Manifest image must be a non-empty relative path")
+    relative = Path(image_name)
+    if relative.is_absolute() or ".." in relative.parts:
+        raise ValueError(f"Manifest image must stay inside images-dir: {image_name}")
+    root = images_dir.resolve()
+    resolved = (root / relative).resolve()
+    try:
+        resolved.relative_to(root)
+    except ValueError as exc:
+        raise ValueError(f"Manifest image must stay inside images-dir: {image_name}") from exc
+    return resolved
+
+
+def select_jsonl_lines(
+    lines: Sequence[str],
+    start_index: Optional[int] = None,
+    end_index: Optional[int] = None,
+    shard_index: Optional[int] = None,
+    num_shards: Optional[int] = None,
+) -> List[str]:
+    uses_range = start_index is not None or end_index is not None
+    uses_shard = shard_index is not None or num_shards is not None
+    if uses_range and uses_shard:
+        raise ValueError("range selection and shard selection are mutually exclusive")
+    if (shard_index is None) != (num_shards is None):
+        raise ValueError("shard_index and num_shards must be set together")
+    if num_shards is not None:
+        if num_shards <= 0:
+            raise ValueError("num_shards must be positive")
+        if shard_index < 0 or shard_index >= num_shards:
+            raise ValueError("shard_index must be in [0, num_shards)")
+
+    selected = list(lines)
+    if uses_range:
+        start = 0 if start_index is None else start_index
+        end = len(selected) if end_index is None else end_index
+        if start < 0 or end < 0:
+            raise ValueError("start_index and end_index must be non-negative")
+        if start > end:
+            raise ValueError("start_index must be <= end_index")
+        if end > len(selected):
+            raise ValueError("end_index must be <= number of records")
+        selected = selected[start:end]
+
+    if num_shards is not None:
+        selected = [line for idx, line in enumerate(selected) if idx % num_shards == shard_index]
+
+    return selected
+
+
+def output_path_for_selection(
+    path: str,
+    start_index: Optional[int] = None,
+    end_index: Optional[int] = None,
+    shard_index: Optional[int] = None,
+    num_shards: Optional[int] = None,
+) -> str:
+    uses_range = start_index is not None or end_index is not None
+    uses_shard = shard_index is not None or num_shards is not None
+    if uses_range and uses_shard:
+        raise ValueError("range selection and shard selection are mutually exclusive")
+    if (shard_index is None) != (num_shards is None):
+        raise ValueError("shard_index and num_shards must be set together")
+
+    suffix = ""
+    if uses_shard:
+        if num_shards <= 0 or shard_index < 0 or shard_index >= num_shards:
+            raise ValueError("invalid shard parameters")
+        width = max(2, len(str(num_shards - 1)))
+        suffix = f".shard{shard_index:0{width}d}-of-{num_shards:0{width}d}"
+    elif uses_range:
+        start = 0 if start_index is None else start_index
+        end = "end" if end_index is None else str(end_index)
+        if start < 0 or (end_index is not None and end_index < start):
+            raise ValueError("invalid range parameters")
+        suffix = f".rows{start}-{end}"
+
+    if not suffix:
+        return path
+    root, extension = os.path.splitext(path)
+    return f"{root}{suffix}{extension or '.json'}"
+
+
+def enrich_ape_results(ape_results: Iterable[dict], image_name: str, extract_image_id) -> List[dict]:
+    image_id = extract_image_id(image_name)
+    enriched = []
+    for ape_item in ape_results:
+        item = dict(ape_item)
+        item["image_id"] = image_id
+        item["category_id"] = item.get("category_name")
+        enriched.append(item)
+    return enriched
+
+
+def write_json_atomic(path: str, payload) -> None:
+    directory = os.path.dirname(os.path.abspath(path))
+    os.makedirs(directory, exist_ok=True)
+    file_descriptor, tmp_path = tempfile.mkstemp(
+        prefix=".orienter-",
+        suffix=".json.tmp",
+        dir=directory,
+    )
+    try:
+        with os.fdopen(file_descriptor, "w", encoding="utf-8") as out_file:
+            json.dump(payload, out_file, indent=4)
+            out_file.write("\n")
+        os.replace(tmp_path, path)
+    except Exception:
+        if os.path.exists(tmp_path):
+            os.unlink(tmp_path)
+        raise
+
+
+def run_optional_reflection(
+    image_path: str,
+    detections: List[dict],
+    detector,
+    enabled: bool = False,
+    model_profile: str = "default",
+    max_iterations: int = 10,
+    advisor=None,
+):
+    if not enabled:
+        return None
+
+    from PIL import Image
+
+    from approach.reflection import openai_compatible_advisor, run_reflection_loop
+
+    def mine_feedback(trace):
+        if not trace:
+            return []
+        return trace[-1].get("feedback", [])
+
+    reflection_advisor = advisor or openai_compatible_advisor(model_profile)
+    with Image.open(image_path) as image:
+        return run_reflection_loop(
+            image.convert("RGB"),
+            list(detections),
+            miner=mine_feedback,
+            detector=detector,
+            advisor=reflection_advisor,
+            max_iterations=max_iterations,
+        )

approach/run_ape.py

@@ -0,0 +1,291 @@
+"""Configurable APE detection entrypoint for the public Orienter pipeline."""
+
+import argparse
+import hashlib
+import json
+import os
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+
+if __package__ in {None, ""}:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from approach.ape_stage import run_ape_stage
+from approach.pipeline_utils import (
+    output_path_for_selection,
+    run_optional_reflection,
+    select_jsonl_lines,
+    write_json_atomic,
+)
+
+
+DEFAULT_APE_CONFIG = (
+    "configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_"
+    "PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_"
+    "cp_16x4_1080k.py"
+)
+APE_CHECKPOINT_SHA256 = "3548f41a3238148180e08fd4b16c71f4abc3ac3caf9c8434444462d1bdb7f965"
+APE_CHECKPOINT_SIZE = 5_956_547_279
+
+
+def _load_jsonl(path: Path):
+    with path.open(encoding="utf-8") as file:
+        return [json.loads(line) for line in file if line.strip()]
+
+
+def _normalize_candidate(record):
+    normalized = dict(record)
+    text = normalized.get("text")
+    if isinstance(text, str):
+        stripped = text.strip()
+        if stripped.startswith("```"):
+            lines = stripped.splitlines()
+            stripped = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
+        normalized["text"] = json.loads(stripped)
+    if normalized.get("text") is not None and "objects" not in normalized["text"]:
+        raise ValueError(
+            f"candidate record {normalized.get('question_id')!r} is missing text.objects"
+        )
+    return normalized
+
+
+@contextmanager
+def _working_directory(path: Path):
+    previous = Path.cwd()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(previous)
+
+
+def _resolve_ape_checkpoint(ape_root: Path, checkpoint: Path) -> Path:
+    if checkpoint.is_absolute():
+        return checkpoint
+    project_root = Path(__file__).resolve().parents[1]
+    repo_relative = (project_root / checkpoint).resolve()
+    if repo_relative.exists():
+        return repo_relative
+    return (ape_root.resolve() / checkpoint).resolve()
+
+
+def _sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _verify_ape_checkpoint(checkpoint: Path, trust_custom_checkpoint: bool = False) -> None:
+    if not checkpoint.is_file():
+        raise FileNotFoundError(f"APE checkpoint does not exist: {checkpoint}")
+
+    size = checkpoint.stat().st_size
+    if size == APE_CHECKPOINT_SIZE:
+        digest = _sha256_file(checkpoint)
+        if digest == APE_CHECKPOINT_SHA256:
+            return
+        mismatch = f"SHA-256 {digest} does not match the released APE-L_D checkpoint"
+    else:
+        mismatch = (
+            f"size {size} does not match the released APE-L_D checkpoint "
+            f"({APE_CHECKPOINT_SIZE} bytes)"
+        )
+
+    if trust_custom_checkpoint:
+        return
+    raise ValueError(
+        f"Refusing unverified checkpoint {checkpoint}: {mismatch}. "
+        "Use --trust-custom-checkpoint only for a checkpoint whose provenance you verified."
+    )
+
+
+def _make_live_inference(args):
+    from approach.ovod.APE.demo.ape_inference import run_ape_model_inference
+
+    ape_root = args.ape_root.resolve()
+    checkpoint = _resolve_ape_checkpoint(ape_root, args.ape_checkpoint)
+    _verify_ape_checkpoint(checkpoint, args.trust_custom_checkpoint)
+    visualization_dir = args.visualization_dir.resolve()
+    visualization_dir.mkdir(parents=True, exist_ok=True)
+
+    def inference(input_path, text_prompt, confidence_threshold):
+        with _working_directory(ape_root):
+            return run_ape_model_inference(
+                config_file=args.ape_config,
+                input_path=input_path,
+                output_path=str(visualization_dir),
+                confidence_threshold=confidence_threshold,
+                text_prompt=text_prompt,
+                with_box=True,
+                with_mask=False,
+                with_sseg=False,
+                opts=[
+                    f"train.init_checkpoint='{checkpoint}'",
+                    "model.model_language.cache_dir=''",
+                    "model.model_vision.select_box_nums_for_evaluation=500",
+                    "model.model_vision.text_feature_bank_reset=True",
+                    "model.model_vision.backbone.net.xattn=False",
+                    "model.model_vision.transformer.encoder.pytorch_attn=True",
+                    "model.model_vision.transformer.decoder.pytorch_attn=True",
+                ],
+            )
+
+    return inference
+
+
+def _merge_detections(retained, new_detections):
+    merged = list(retained)
+    seen = {
+        (item.get("category_name"), tuple(item.get("bbox", [])))
+        for item in retained
+    }
+    for item in new_detections:
+        key = item.get("category_name"), tuple(item.get("bbox", []))
+        if key not in seen:
+            merged.append(item)
+            seen.add(key)
+    return merged
+
+
+def build_parser():
+    project_root = Path(__file__).resolve().parent
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--questions", type=Path, required=True, help="question manifest JSONL")
+    parser.add_argument("--candidates", type=Path, required=True, help="candidate JSONL")
+    parser.add_argument("--images-dir", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--start-index", type=int)
+    parser.add_argument("--end-index", type=int)
+    parser.add_argument("--shard-index", type=int)
+    parser.add_argument("--num-shards", type=int)
+    parser.add_argument("--resume", action="store_true")
+    parser.add_argument("--threshold", type=float, default=0.15)
+    parser.add_argument(
+        "--ape-root",
+        type=Path,
+        default=project_root / "ovod" / "APE",
+    )
+    parser.add_argument("--ape-config", default=DEFAULT_APE_CONFIG)
+    parser.add_argument(
+        "--ape-checkpoint",
+        type=Path,
+        default=Path("ape_d_model_final.pth"),
+    )
+    parser.add_argument(
+        "--trust-custom-checkpoint",
+        action="store_true",
+        help="allow a checkpoint that does not match the released model hash",
+    )
+    parser.add_argument(
+        "--visualization-dir",
+        type=Path,
+        default=Path("outputs/ape_visualizations"),
+    )
+    parser.add_argument("--enable-reflection", action="store_true")
+    parser.add_argument("--reflection-profile", default="default")
+    parser.add_argument("--max-reflection-iterations", type=int, default=10)
+    return parser
+
+
+def run(args, inference=None):
+    questions = {
+        item["question_id"]: item
+        for item in _load_jsonl(args.questions)
+    }
+    records = [_normalize_candidate(item) for item in _load_jsonl(args.candidates)]
+    selected = select_jsonl_lines(
+        records,
+        start_index=args.start_index,
+        end_index=args.end_index,
+        shard_index=args.shard_index,
+        num_shards=args.num_shards,
+    )
+    selected_output = Path(
+        output_path_for_selection(
+            str(args.output),
+            start_index=args.start_index,
+            end_index=args.end_index,
+            shard_index=args.shard_index,
+            num_shards=args.num_shards,
+        )
+    )
+    base_inference = inference or _make_live_inference(args)
+    reflection_traces = []
+
+    def configured_inference(**kwargs):
+        detections = base_inference(**kwargs)
+        if not args.enable_reflection:
+            return detections
+
+        def redetect(candidates, retained):
+            if not candidates:
+                return retained
+            new_detections = base_inference(
+                input_path=kwargs["input_path"],
+                text_prompt=", ".join(candidates),
+                confidence_threshold=kwargs["confidence_threshold"],
+            )
+            return _merge_detections(retained, new_detections)
+
+        try:
+            reflection = run_optional_reflection(
+                kwargs["input_path"],
+                detections,
+                detector=redetect,
+                enabled=True,
+                model_profile=args.reflection_profile,
+                max_iterations=args.max_reflection_iterations,
+            )
+        except Exception as exc:
+            reflection_traces.append(
+                {
+                    "image": Path(kwargs["input_path"]).name,
+                    "error_type": type(exc).__name__,
+                    "message": str(exc),
+                }
+            )
+            return detections
+
+        reflection_traces.append(
+            {
+                "image": Path(kwargs["input_path"]).name,
+                "trace": reflection["trace"],
+                "max_iterations_reached": reflection["max_iterations_reached"],
+            }
+        )
+        return reflection["detections"]
+
+    error_path = selected_output.with_suffix(".errors.json")
+    results, errors = run_ape_stage(
+        records=selected,
+        questions=questions,
+        images_dir=args.images_dir,
+        output_path=selected_output,
+        inference=configured_inference,
+        inference_kwargs={"confidence_threshold": args.threshold},
+        error_path=error_path,
+        resume=args.resume,
+    )
+    if args.enable_reflection:
+        write_json_atomic(
+            str(selected_output.with_suffix(".reflection.json")),
+            reflection_traces,
+        )
+    return {
+        "output": str(selected_output),
+        "records_selected": len(selected),
+        "predictions": len(results),
+        "errors": len(errors),
+    }
+
+
+def main(argv=None):
+    args = build_parser().parse_args(argv)
+    print(json.dumps(run(args), indent=2))
+
+
+if __name__ == "__main__":
+    main()

approach/util/test.py

@@ -0,0 +1,35 @@
+import torch 
+import time
+import os
+import argparse
+import shutil
+import sys
+ 
+def parse_args():
+    parser = argparse.ArgumentParser(description='Matrix multiplication')
+    parser.add_argument('--gpus', help='gpu amount', required=True, type=int)
+    parser.add_argument('--size', help='matrix size', required=True, type=int)
+    parser.add_argument('--interval', help='sleep interval', required=True, type=float)
+    args = parser.parse_args()
+    return args
+ 
+ 
+def matrix_multiplication(args):
+ 
+    a_list, b_list, result = [], [], []    
+    size = (args.size, args.size)
+    
+    for i in range(args.gpus):
+        a_list.append(torch.rand(size, device=i))
+        b_list.append(torch.rand(size, device=i))
+        result.append(torch.rand(size, device=i))
+ 
+    while True:
+        for i in range(args.gpus):
+            result[i] = a_list[i] * b_list[i]
+        time.sleep(args.interval)
+ 
+if __name__ == "__main__":
+    # usage: python test.py --size 32000 --gpus 4 --interval 0.01
+    args = parse_args()
+    matrix_multiplication(args)

approach/vlm/LLaVA/.gitignore

@@ -0,0 +1,29 @@
+# Python
+__pycache__
+*.pyc
+*.egg-info
+dist
+
+# Log
+*.log
+*.log.*
+*.json
+*.jsonl
+
+# Data
+!**/alpaca-data-conversation.json
+
+# Editor
+.idea
+*.swp
+
+# Other
+.DS_Store
+wandb
+output
+
+checkpoints
+ckpts*
+
+.ipynb_checkpoints
+*.ipynb

approach/vlm/LLaVA/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

approach/vlm/LLaVA/README.md

@@ -0,0 +1,366 @@
+# 🌋 LLaVA: Large Language and Vision Assistant
+
+*Visual instruction tuning towards large language and vision models with GPT-4 level capabilities.*
+
+[[Project Page](https://llava-vl.github.io/)] [[Demo](https://llava.hliu.cc/)]  [[Data](https://github.com/haotian-liu/LLaVA/blob/main/docs/Data.md)] [[Model Zoo](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md)]
+
+**Improved Baselines with Visual Instruction Tuning** [[Paper](https://arxiv.org/abs/2310.03744)] <br>
+[Haotian Liu](https://hliu.cc), [Chunyuan Li](https://chunyuan.li/), [Yuheng Li](https://yuheng-li.github.io/), [Yong Jae Lee](https://pages.cs.wisc.edu/~yongjaelee/)
+
+**Visual Instruction Tuning** (NeurIPS 2023, **Oral**) [[Paper](https://arxiv.org/abs/2304.08485)]<br>
+[Haotian Liu*](https://hliu.cc), [Chunyuan Li*](https://chunyuan.li/), [Qingyang Wu](https://scholar.google.ca/citations?user=HDiw-TsAAAAJ&hl=en/), [Yong Jae Lee](https://pages.cs.wisc.edu/~yongjaelee/) (*Equal Contribution)
+
+<p align="center">
+    <a href="https://llava.hliu.cc/"><img src="images/llava_logo.png" width="50%"></a> <br>
+    Generated by <a href="https://gligen.github.io/">GLIGEN</a> via "a cute lava llama with glasses" and box prompt
+</p>
+
+
+## Release
+- [10/5] 🔥 LLaVA-1.5 is out! Achieving SoTA on 11 benchmarks, with just simple modifications to the original LLaVA, utilizes all public data, completes training in ~1 day on a single 8-A100 node, and surpasses methods like Qwen-VL-Chat that use billion-scale data. Check out the [technical report](https://arxiv.org/abs/2310.03744), and explore the [demo](https://llava.hliu.cc/)! Models are available in [Model Zoo](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md), with training and evaluation scripts coming in the next week!
+- [9/26] LLaVA is improved with reinforcement learning from human feedback (RLHF) to improve fact grounding and reduce hallucination. Check out the new SFT and RLHF checkpoints at project [[LLavA-RLHF]](https://llava-rlhf.github.io/)
+- [9/22] [LLaVA](https://arxiv.org/abs/2304.08485) is accpeted by NeurIPS 2023 as **oral presentation**, and [LLaVA-Med](https://arxiv.org/abs/2306.00890) is accpeted by NeurIPS 2023 Datasets and Benchmarks Track as **spotlight presentation**.
+- [9/20] We summarize our emprical study of training 33B and 65B LLaVA mdoels in a [note](https://arxiv.org/abs/2309.09958). Further, if you are interested in the comprehensive review, evolution and trend of multimodal foundation models, please check out our recent survey paper [``Multimodal Foundation Models: From Specialists to General-Purpose Assistants''.](https://arxiv.org/abs/2309.10020)
+<p align="center">
+  <img src="https://github.com/Computer-Vision-in-the-Wild/CVinW_Readings/blob/main/images/mfm_evolution.jpeg?raw=true" width=50%/>
+</p>  
+
+- [7/19] 🔥 We release a major upgrade, including support for LLaMA-2, LoRA training, 4-/8-bit inference, higher resolution (336x336), and a lot more. We release [LLaVA Bench](https://github.com/haotian-liu/LLaVA/blob/main/docs/LLaVA_Bench.md) for benchmarking open-ended visual chat with results from Bard and Bing-Chat. We also support and verify training with RTX 3090 and RTX A6000. Check out [LLaVA-from-LLaMA-2](https://github.com/haotian-liu/LLaVA/blob/main/docs/LLaVA_from_LLaMA2.md), and our [model zoo](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md)!
+- [6/26] [CVPR 2023 Tutorial](https://vlp-tutorial.github.io/) on **Large Multimodal Models: Towards Building and Surpassing Multimodal GPT-4**!  Please check out [[Slides](https://datarelease.blob.core.windows.net/tutorial/vision_foundation_models_2023/slides/Chunyuan_cvpr2023_tutorial_lmm.pdf)] [[Notes](https://arxiv.org/abs/2306.14895)] [[YouTube](https://youtu.be/mkI7EPD1vp8)] [[Bilibli](https://www.bilibili.com/video/BV1Ng4y1T7v3/)].
+- [6/11] We released the preview for the most requested feature: DeepSpeed and LoRA support!  Please see documentations [here](./docs/LoRA.md).
+- [6/1] We released **LLaVA-Med: Large Language and Vision Assistant for Biomedicine**, a step towards building biomedical domain large language and vision models with GPT-4 level capabilities.  Checkout the [paper](https://arxiv.org/abs/2306.00890) and [page](https://github.com/microsoft/LLaVA-Med).
+- [5/6] We are releasing [LLaVA-Lighting-MPT-7B-preview](https://huggingface.co/liuhaotian/LLaVA-Lightning-MPT-7B-preview), based on MPT-7B-Chat!  See [here](#LLaVA-MPT-7b) for more details.
+- [5/2] 🔥 We are releasing LLaVA-Lighting!  Train a lite, multimodal GPT-4 with just $40 in 3 hours!  See [here](#train-llava-lightning) for more details.
+- [4/27] Thanks to the community effort, LLaVA-13B with 4-bit quantization allows you to run on a GPU with as few as 12GB VRAM!  Try it out [here](https://github.com/oobabooga/text-generation-webui/tree/main/extensions/llava).
+- [4/17] 🔥 We released **LLaVA: Large Language and Vision Assistant**. We propose visual instruction tuning, towards building large language and vision models with GPT-4 level capabilities.  Checkout the [paper](https://arxiv.org/abs/2304.08485) and [demo](https://llava.hliu.cc/).
+
+<!-- <a href="https://llava.hliu.cc/"><img src="assets/demo.gif" width="70%"></a> -->
+
+[![Code License](https://img.shields.io/badge/Code%20License-Apache_2.0-green.svg)](https://github.com/tatsu-lab/stanford_alpaca/blob/main/LICENSE)
+[![Data License](https://img.shields.io/badge/Data%20License-CC%20By%20NC%204.0-red.svg)](https://github.com/tatsu-lab/stanford_alpaca/blob/main/DATA_LICENSE)
+**Usage and License Notices**: The data and checkpoint is intended and licensed for research use only. They are also restricted to uses that follow the license agreement of LLaMA, Vicuna and GPT-4. The dataset is CC BY NC 4.0 (allowing only non-commercial use) and models trained using the dataset should not be used outside of research purposes.
+
+
+## Contents
+- [Install](#install)
+- [LLaVA Weights](#llava-weights)
+- [Demo](#Demo)
+- [Model Zoo](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md)
+- [Dataset](https://github.com/haotian-liu/LLaVA/blob/main/docs/Data.md)
+- [Train](#train)
+- [Evaluation](#evaluation)
+
+## Install
+
+1. Clone this repository and navigate to LLaVA folder
+```bash
+git clone https://github.com/haotian-liu/LLaVA.git
+cd LLaVA
+```
+
+2. Install Package
+```Shell
+conda create -n llava python=3.10 -y
+conda activate llava
+pip install --upgrade pip  # enable PEP 660 support
+pip install -e .
+```
+
+3. Install additional packages for training cases
+```
+pip install ninja
+pip install flash-attn --no-build-isolation
+```
+
+### Upgrade to latest code base
+
+```Shell
+git pull
+pip uninstall transformers
+pip install -e .
+```
+
+## LLaVA Weights
+Please check out our [Model Zoo](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md) for all public LLaVA checkpoints, and the instructions of how to use the weights.
+
+## Demo
+
+To run our demo, you need to prepare LLaVA checkpoints locally.  Please follow the instructions [here](#llava-weights) to download the checkpoints.
+
+### Gradio Web UI
+
+To launch a Gradio demo locally, please run the following commands one by one. If you plan to launch multiple model workers to compare between different checkpoints, you only need to launch the controller and the web server *ONCE*.
+
+#### Launch a controller
+```Shell
+python -m llava.serve.controller --host 0.0.0.0 --port 10000
+```
+
+#### Launch a gradio web server.
+```Shell
+python -m llava.serve.gradio_web_server --controller http://localhost:10000 --model-list-mode reload
+```
+You just launched the Gradio web interface. Now, you can open the web interface with the URL printed on the screen. You may notice that there is no model in the model list. Do not worry, as we have not launched any model worker yet. It will be automatically updated when you launch a model worker.
+
+#### Launch a model worker
+
+This is the actual *worker* that performs the inference on the GPU.  Each worker is responsible for a single model specified in `--model-path`.
+
+```Shell
+python -m llava.serve.model_worker --host 0.0.0.0 --controller http://localhost:10000 --port 40000 --worker http://localhost:40000 --model-path liuhaotian/llava-v1.5-13b
+```
+Wait until the process finishes loading the model and you see "Uvicorn running on ...".  Now, refresh your Gradio web UI, and you will see the model you just launched in the model list.
+
+You can launch as many workers as you want, and compare between different model checkpoints in the same Gradio interface. Please keep the `--controller` the same, and modify the `--port` and `--worker` to a different port number for each worker.
+```Shell
+python -m llava.serve.model_worker --host 0.0.0.0 --controller http://localhost:10000 --port <different from 40000, say 40001> --worker http://localhost:<change accordingly, i.e. 40001> --model-path <ckpt2>
+```
+
+If you are using an Apple device with an M1 or M2 chip, you can specify the mps device by using the `--device` flag: `--device mps`.
+
+#### Launch a model worker (Multiple GPUs, when GPU VRAM <= 24GB)
+
+If the VRAM of your GPU is less than 24GB (e.g., RTX 3090, RTX 4090, etc.), you may try running it with multiple GPUs. Our latest code base will automatically try to use multiple GPUs if you have more than one GPU. You can specify which GPUs to use with `CUDA_VISIBLE_DEVICES`. Below is an example of running with the first two GPUs.
+
+```Shell
+CUDA_VISIBLE_DEVICES=0,1 python -m llava.serve.model_worker --host 0.0.0.0 --controller http://localhost:10000 --port 40000 --worker http://localhost:40000 --model-path liuhaotian/llava-v1.5-13b
+```
+
+#### Launch a model worker (4-bit, 8-bit inference, quantized)
+
+You can launch the model worker with quantized bits (4-bit, 8-bit), which allows you to run the inference with reduced GPU memory footprint, potentially allowing you to run on a GPU with as few as 12GB VRAM. Note that inference with quantized bits may not be as accurate as the full-precision model. Simply append `--load-4bit` or `--load-8bit` to the **model worker** command that you are executing. Below is an example of running with 4-bit quantization.
+
+```Shell
+python -m llava.serve.model_worker --host 0.0.0.0 --controller http://localhost:10000 --port 40000 --worker http://localhost:40000 --model-path liuhaotian/llava-v1.5-13b --load-4bit
+```
+
+#### Launch a model worker (LoRA weights, unmerged)
+
+You can launch the model worker with LoRA weights, without merging them with the base checkpoint, to save disk space. There will be additional loading time, while the inference speed is the same as the merged checkpoints. Unmerged LoRA checkpoints do not have `lora-merge` in the model name, and are usually much smaller (less than 1GB) than the merged checkpoints (13G for 7B, and 25G for 13B).
+
+To load unmerged LoRA weights, you simply need to pass an additional argument `--model-base`, which is the base LLM that is used to train the LoRA weights. You can check the base LLM of each LoRA weights in the [model zoo](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md).
+
+```Shell
+python -m llava.serve.model_worker --host 0.0.0.0 --controller http://localhost:10000 --port 40000 --worker http://localhost:40000 --model-path liuhaotian/llava-v1-0719-336px-lora-vicuna-13b-v1.3 --model-base lmsys/vicuna-13b-v1.3
+```
+
+### CLI Inference
+
+Chat about images using LLaVA without the need of Gradio interface. It also supports multiple GPUs, 4-bit and 8-bit quantized inference. With 4-bit quantization, for our LLaVA-1.5-7B, it uses less than 8GB VRAM on a single GPU.
+
+```Shell
+python -m llava.serve.cli \
+    --model-path liuhaotian/llava-v1.5-7b \
+    --image-file "https://llava-vl.github.io/static/images/view.jpg" \
+    --load-4bit
+```
+
+<img src="images/demo_cli.gif" width="70%">
+
+## Train
+
+LLaVA training consists of two stages: (1) feature alignment stage: use approximately 600K filtered CC3M to connect a *frozen pretrained* vision encoder to a *frozen LLM*; (2) visual instruction tuning stage: use 150K GPT-generated multimodal instruction-following to teach the model to follow multimodal instructions.
+
+LLaVA is trained on 8 A100 GPUs with 80GB memory. To train on fewer GPUs, you can reduce the `per_device_train_batch_size` and increase the `gradient_accumulation_steps` accordingly. Always keep the global batch size the same: `per_device_train_batch_size` x `gradient_accumulation_steps` x `num_gpus`.
+
+### Hyperparameters
+We use a similar set of hyperparameters as Vicuna in finetuning.  Both hyperparameters used in pretraining and finetuning are provided below.
+
+1. Pretraining
+
+| Hyperparameter | Global Batch Size | Learning rate | Epochs | Max length | Weight decay |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| LLaVA-13B | 256 | 1e-3 | 1 | 2048 | 0 |
+
+2. Finetuning
+
+| Hyperparameter | Global Batch Size | Learning rate | Epochs | Max length | Weight decay |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| LLaVA-13B | 128 | 2e-5 | 1 | 2048 | 0 |
+
+### Prepare Vicuna checkpoints
+
+Before you start, prepare our base model Vicuna, which is an instruction-tuned chatbot. Please download its weights [here](https://github.com/lm-sys/FastChat#model-weights).
+
+Vicuna has two versions: v0 and v1, the main difference between them is the prompt of format. We support both. To ensure the best performance, you need to specify the correct prompt version corresponding to the weights you download: `v0` for `v0` weights, and `v1` for all Vicuna `v1.x` models.
+
+### Pretrain (feature alignment)
+
+Please download the subset of the CC3M dataset we use in the paper [here](https://huggingface.co/datasets/liuhaotian/LLaVA-CC3M-Pretrain-595K).
+
+Pretrain takes around 4 hours for LLaVA-13B on 8x A100 (80G). It takes around 2 hours for 7B checkpoints.
+
+We recommend training with DeepSpeed as it can save a lot of GPU RAM. We provide training script with DeepSpeed [here](https://github.com/haotian-liu/LLaVA/blob/main/scripts/pretrain.sh).
+
+You may run this with a single A100 GPU with the following code.  Please note that the `per_device_train_batch_size` * `gradient_accumulation_steps` should be equal to 128 to keep the global batch size the same.
+
+<details>
+<summary>Pretrain: LLaVA-13B, 1x A100 (80G).  Time: ~33 hours.</summary>
+
+```Shell
+python llava/train/train_mem.py \
+    --model_name_or_path ./checkpoints/vicuna-13b \
+    --version [v0 or v1] \
+    --data_path /path/to/cc3m_595k.json \
+    --image_folder /path/to/cc3m_595k_images \
+    --vision_tower openai/clip-vit-large-patch14 \
+    --tune_mm_mlp_adapter True \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
+    --output_dir ./checkpoints/llava-13b-pretrain \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 8 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 2400 \
+    --save_total_limit 1 \
+    --learning_rate 2e-3 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --lazy_preprocess True \
+    --report_to wandb
+```
+</details>
+
+
+### Visual Instruction Tuning
+
+1. Prepare data
+
+Please download the annotation of our instruction tuning data [llava_instruct_158k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/llava_instruct_150k.json), and download the COCO train2017 images [here](https://cocodataset.org/#download).
+
+2. Start training!
+
+You may download our pretrained projectors in [Model Zoo](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md). It is not recommended to use legacy projectors, as they may be trained with a different version of the codebase, and if any option is off, the model will not function/train as we expected.
+
+When we initially released our paper, we used a full 3-epoch schedule on the LLaVA-Instruct-158K dataset. The scripts are provided [here](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune_full_schedule.sh).
+
+In our later exploration, we introduced LLaVA-Lightning, as we find that a much faster 1-epoch schedule on LLaVA-Instruct-80K can achieve fast convergence and good performance. With LLaVA Lightning, we are able to train, validate, and release LLaVA-LLaMA-2 checkpoints preview on the same day as LLaMA-2 release. If you are interested to learn more about LLaVA Lightning, please continue to the following section.
+
+### Lightning
+
+LLaVA-Lightning can be trained on 8x A100 GPUs in just 3 hours, including both pretraining and finetuning. When using spot instances, it costs just ~$40.
+
+For LLaVA Lightning, we create two distilled subset to ensure both a broad concept coverage, and the efficiency in training. Furthermore, we only perform instruction tuning for 1 epoch, in contrast to 3 epochs in the paper. We find such schedule is effective and can achieve fast convergence and good performance.
+
+For pretraining, we create a concept-balanced subset of LAION-CC-SBU. It consists of 558K images.  Download data [here](https://huggingface.co/datasets/liuhaotian/LLaVA-Pretrain/tree/main).
+
+For instruction tuning, we create a subset of LLaVA-Instruct-150K. It consists of 80K image-instruction pairs, consisting of 40K conversation and 40K complex reasoning data, with non-overlapping images. Download `llava_instruct_80k.json` [here](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/llava_instruct_80k.json).
+
+#### Hyperparameters
+
+1. Pretraining ([script](https://github.com/haotian-liu/LLaVA/blob/main/scripts/pretrain.sh))
+
+| Hyperparameter | Global Batch Size | Learning rate | Epochs | Max length | Weight decay |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| LLaVA-Lightning | 128 | 2e-3 | 1 | 2048 | 0 |
+
+2. Visual Instruction Tuning ([script](https://github.com/haotian-liu/LLaVA/blob/main/scripts/finetune.sh))
+
+| Hyperparameter | Global Batch Size | Learning rate | Epochs | Max length | Weight decay |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| LLaVA-Lightning | 128 | 2e-5 | 1 | 2048 | 0 |
+
+#### LLaVA-MPT-7b
+Thanks to LLaVA-Lightning, we are able to train a checkpoint based on MPT-7B-Chat on 8x A100 GPUs in just 3 hours, including both pretraining and finetuning.
+
+**NOTE**: This is a research preview of the LLaVA-Lightning based on MPT-7B-chat checkpoint. The usage of the model should comply with MPT-7B-chat license and agreements.
+
+1. Usage
+
+You do not need to download our checkpoint, it will directly load from our Hugging Face model: [`liuhaotian/LLaVA-Lightning-MPT-7B-preview`](https://huggingface.co/liuhaotian/LLaVA-Lightning-MPT-7B-preview).
+
+```Shell
+python -m llava.serve.controller --host 0.0.0.0 --port 10000
+python -m llava.serve.model_worker --host 0.0.0.0 --controller http://localhost:10000 --port 40000 --worker http://localhost:40000 --model-path liuhaotian/LLaVA-Lightning-MPT-7B-preview
+python -m llava.serve.gradio_web_server --controller http://localhost:10000
+```
+
+2. Training
+
+We use the same set of training dataset, and the hyperparameters as other *Lightning* checkpoints.
+
+## Evaluation
+
+### GPT-assisted Evaluation
+
+Our GPT-assisted evaluation pipeline for multimodal modeling is provided for a comprehensive understanding of the capabilities of vision-language models.  Please see our paper for more details.
+
+1. Generate LLaVA responses
+
+```Shell
+python model_vqa.py \
+    --model-path ./checkpoints/LLaVA-13B-v0 \
+    --question-file \
+    playground/data/coco2014_val_qa_eval/qa90_questions.jsonl \
+    --image-folder \
+    /path/to/coco2014_val \
+    --answers-file \
+    /path/to/answer-file-our.jsonl
+```
+
+2. Evaluate the generated responses.  In our case, [`answer-file-ref.jsonl`](./playground/data/coco2014_val_qa_eval/qa90_gpt4_answer.jsonl) is the response generated by text-only GPT-4 (0314), with the context captions/boxes provided.
+
+```Shell
+OPENAI_API_KEY="sk-***********************************" python llava/eval/eval_gpt_review_visual.py \
+    --question playground/data/coco2014_val_qa_eval/qa90_questions.jsonl \
+    --context llava/eval/table/caps_boxes_coco2014_val_80.jsonl \
+    --answer-list \
+    /path/to/answer-file-ref.jsonl \
+    /path/to/answer-file-our.jsonl \
+    --rule llava/eval/table/rule.json \
+    --output /path/to/review.json
+```
+
+3. Summarize the evaluation results
+
+```Shell
+python summarize_gpt_review.py
+```
+
+## ScienceQA
+
+Please check out the documentation [here](https://github.com/haotian-liu/LLaVA/blob/main/docs/ScienceQA.md).
+
+## Citation
+
+If you find LLaVA useful for your research and applications, please cite using this BibTeX:
+```bibtex
+
+@misc{liu2023improvedllava,
+      title={Improved Baselines with Visual Instruction Tuning}, 
+      author={Liu, Haotian and Li, Chunyuan and Li, Yuheng and Lee, Yong Jae},
+      publisher={arXiv:2310.03744},
+      year={2023},
+}
+
+@misc{liu2023llava,
+      title={Visual Instruction Tuning}, 
+      author={Liu, Haotian and Li, Chunyuan and Wu, Qingyang and Lee, Yong Jae},
+      publisher={arXiv:2304.08485},
+      year={2023},
+}
+```
+
+## Acknowledgement
+
+- [Vicuna](https://github.com/lm-sys/FastChat): the codebase we built upon, and our base model Vicuna-13B that has the amazing language capabilities!
+
+## Related Projects
+
+- [Instruction Tuning with GPT-4](https://github.com/Instruction-Tuning-with-GPT-4/GPT-4-LLM)
+- [LLaVA-Med: Training a Large Language-and-Vision Assistant for Biomedicine in One Day](https://github.com/microsoft/LLaVA-Med)
+- [Otter: In-Context Multi-Modal Instruction Tuning](https://github.com/Luodian/Otter)
+
+For future project ideas, please check out:
+- [SEEM: Segment Everything Everywhere All at Once](https://github.com/UX-Decoder/Segment-Everything-Everywhere-All-At-Once)
+- [Grounded-Segment-Anything](https://github.com/IDEA-Research/Grounded-Segment-Anything) to detect, segment, and generate anything by marrying [Grounding DINO](https://github.com/IDEA-Research/GroundingDINO) and [Segment-Anything](https://github.com/facebookresearch/segment-anything).

approach/vlm/LLaVA/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "llava"
+version = "1.1.0"
+description = "Towards GPT-4 like large language and visual assistant."
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache Software License",
+]
+dependencies = [
+    "einops", "fastapi", "gradio==3.35.2", "markdown2[all]", "numpy",
+    "requests", "sentencepiece", "tokenizers>=0.12.1",
+    "torch==2.0.1", "torchvision==0.15.2", "uvicorn", "wandb",
+    "shortuuid", "httpx==0.24.0",
+    "deepspeed==0.9.5",
+    "peft==0.4.0",
+    "transformers==4.31.0",
+    "accelerate==0.21.0",
+    "bitsandbytes==0.41.0",
+    "scikit-learn==1.2.2",
+    "sentencepiece==0.1.99",
+    "einops==0.6.1", "einops-exts==0.0.4", "timm==0.6.13",
+    "gradio_client==0.2.9"
+]
+
+[project.urls]
+"Homepage" = "https://llava-vl.github.io"
+"Bug Tracker" = "https://github.com/haotian-liu/LLaVA/issues"
+
+[tool.setuptools.packages.find]
+exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]
+
+[tool.wheel]
+exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]

approach/vlm/gpt4v/cu_gpt4v.py

@@ -0,0 +1,25 @@
+"""Compatibility wrapper for the former CUHK-specific GPT-4V adapter."""
+
+import os
+
+from approach.vlm.gpt4v.gpt4v import (
+    encode_image,
+    generate_gpt4v_prompt,
+    get_steam_app_data,
+    process_image as _process_image,
+    process_image_q,
+)
+
+
+def process_image(image_question, image_path, ablation, key_idx=0):
+    profile = os.environ.get("ORIENT_MODEL_PROFILE", "default")
+    return _process_image(profile, image_question, image_path, ablation, key_idx)
+
+
+__all__ = [
+    "encode_image",
+    "generate_gpt4v_prompt",
+    "get_steam_app_data",
+    "process_image",
+    "process_image_q",
+]

approach/vlm/gpt4v/gpt4v.py

@@ -0,0 +1,148 @@
+import os
+import base64
+import mimetypes
+import requests
+import logging
+from bs4 import BeautifulSoup
+from approach.app_metadata import get_app_metadata, load_app_metadata_cache
+from approach.config import get_model_profile
+from approach.providers import EncodedImage, OpenAICompatibleChatClient
+
+DEFAULT_PROFILE = os.environ.get("ORIENT_MODEL_PROFILE", "default")
+
+
+def _app_metadata_from_cache(app_id, metadata_cache):
+    return get_app_metadata(app_id, metadata_cache)
+
+
+def get_steam_app_data(app_id, image_name, metadata_cache=None):
+    if metadata_cache is not None:
+        return _app_metadata_from_cache(app_id, metadata_cache)
+
+    if app_id == '1718650':
+        app_name = 'Nuremberg: VRdict of Nations'
+        app_description = 'A VR investigation of the crimes against humanity committed by the Nazi leaders. This is a detective story in VR, a documentary investigation, brought to you by Rossiya Segodnya. Your goal is to find and collect evidence to prove that the key Nazi criminals and the leaders of the Third Reich are guilty. Today, less than 50% of young people are familiar with the historic Nuremberg trial, its process and results (according to a survey conducted by the “Nuremberg: Casus Pacis” project). The charges and verdicts brought forward by the Nuremberg tribunal, its fairness or even necessity are often questioned. This happens out of ignorance or under pressure from those who benefit from distorting the historical facts. Your virtual journey begins in the Spandau Prison dining area recreated from one of the legendary post-war photos. Here, you are met by serenely dining Nuremberg defendants – Göring, Dönitz, Rosenberg, von Ribbentrop and von Schirach. By touching each of them, you can travel back to these Nazi criminals’ past. Your task is to find evidence of their terrible crimes against humanity in – seemingly – an ordinary and peaceful environment. As you gather more pieces of evidence, you put together a convincing dossier, akin to those ones that the 1946 Nuremberg verdict was based on. Thus you will restore the historical truth.'
+        return app_name, app_description
+
+    url = f"https://store.steampowered.com/app/{app_id}"
+
+    # proxies = {
+    #     "http": f"http://{proxy}",
+    #     "https": f"http://{proxy}"
+    # }
+
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
+    }
+
+    try:
+        response = requests.get(url, headers=headers, timeout=30)
+        response.raise_for_status()
+
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        app_name = soup.find("div", {"id": "appHubAppName"}).get_text(strip=True)
+        short_desc = soup.find("div", class_="game_description_snippet").get_text(strip=True)
+        long_desc = soup.find("div", {"id": "game_area_description"}).get_text(strip=True)
+
+        return app_name, short_desc + " " + long_desc
+    except Exception as e:
+        # print(f"Error fetching app data for app_id {app_id} of image {image_name}. Error: {e}")
+        logging.error(f"Error fetching app data for app_id {app_id} of image {image_name}. Error: {e}")
+        return "", ""
+
+# Function to encode the image
+def encode_image(image_path):
+  with open(image_path, "rb") as image_file:
+    return base64.b64encode(image_file.read()).decode('utf-8')
+
+
+def generate_gpt4v_prompt(app_name, app_description, image_question, ablation):
+    if ablation:
+        # ISSTA version
+        # return 'Describe the image.'
+
+        # ICSE 2025 version
+        return f'''
+        --- Context and Task ---
+        You are a virtual reality game player.
+        Currently, you are playing a game named {app_name} with the following description. Now in your field of view from the VR headset, you can see the VR scenario as the uploaded screenshot figure.
+        Please write referring expressions for all objects in the scenario.
+        For referring expressions, describe all characteristics from different perspectives that make the object distinguishable from other objects such as <size, color, shape and type>.
+        DO NOT MENTION LOCATION INFORMATION OR MENTION OTHER OBJECTS IN REFERRING EXPRESSIONS!\n
+
+        --- Instructions and Demonstrations ---
+        Ignore user-controlled VR device objects, e.g., controllers, hands and gloves, which are near the users in screenshot.\n
+
+        --- Output Format ---
+        Output in JSON format: {{"objects": {{"here_is_object_name": referring_expression_1, "here_is_object_name": referring_expression_2}}}}. NO DOT OUTPUT any other content besides JSON.\n
+
+        --- App Description Information ---
+        The description of the app:
+        {app_description}.
+        '''
+    elif not ablation:
+        # ISSTA 2024 version
+        # return f'''
+        # You are a virtual reality game player. Currently, you are playing a game named {app_name} with the following description and get the screenshot I uploaded. Please describe all objects in the screenshot.
+
+        # The description of the app:
+        # {app_description}.
+        # '''
+
+        # ICSE 2025 version
+        return f'''
+        --- Context and Task ---
+        You are a virtual reality game player.
+        Currently, you are playing a game named {app_name} with the following description. Now in your field of view from the VR headset, you can see the VR scenario as the uploaded screenshot figure.
+        Please: (1) identify all user-interactable objects in this screenshot, with which you can use VR devices like handheld controllers to interact; and (2) write referring expressions for user-interactable objects.
+        For referring expressions, describe all characteristics from different perspectives that make the object distinguishable from other objects such as <size, color, shape and type>.
+        DO NOT MENTION LOCATION INFORMATION OR MENTION OTHER OBJECTS IN REFERRING EXPRESSIONS!\n
+
+        --- Instructions and Demonstrations ---
+        (1) For a tree object in a tree-planting game, the users need to pick up and plant the tree, thus the object is interactable.
+        (2) But for a tree object appeard in the background scenery of a fishing-only game, the object is highly likely non-interactable.
+        (3) Ignore user-controlled VR device objects, e.g., controllers, hands and gloves, which are near the users in screenshot.\n
+
+        --- Output Format ---
+        Output in JSON format: {{"objects": {{"here_is_object_name": referring_expression_1, "here_is_object_name": referring_expression_2}}}}. NO DOT OUTPUT any other content besides JSON.\n
+
+        --- App Description Information ---
+        The description of the app:
+        {app_description}.
+        '''
+
+def process_image_q(image_question, image_path, key_idx, metadata_cache=None):
+    # time.sleep(2)
+    app_id = image_path.split('/')[-1].split("_")[0]
+    # print(app_id)
+    app_name, app_description = get_steam_app_data(app_id, image_path, metadata_cache)
+
+    gpt4v_prompt = generate_gpt4v_prompt(app_name, app_description, image_question, True)
+
+    return gpt4v_prompt
+
+
+
+def process_image(vlm, image_question, image_path, ablation, key_idx, metadata_cache=None):
+    app_id = image_path.split('/')[-1].split("_")[0]
+    # print(app_id)
+    app_name, app_description = get_steam_app_data(app_id, image_path, metadata_cache)
+
+    # Getting the base64 string
+    base64_image = encode_image(image_path)
+
+    gpt4v_prompt = generate_gpt4v_prompt(app_name, app_description, image_question, ablation)
+    # gpt4v_prompt = image_question
+
+    try:
+        profile = get_model_profile(vlm or DEFAULT_PROFILE)
+        media_type = mimetypes.guess_type(image_path)[0] or "image/jpeg"
+        client = OpenAICompatibleChatClient(profile)
+        return client.complete_json(
+            gpt4v_prompt,
+            [EncodedImage(base64_image, media_type)],
+        )
+    except Exception as e:
+        logging.error(f"Error generating completion for image {image_path}. Error: {e}")
+        return ''

configs/model_profiles.yaml

@@ -0,0 +1,31 @@
+profiles:
+  default:
+    provider: openrouter
+    model: openai/gpt-5.6-sol
+    base_url: https://openrouter.ai/api/v1
+    api_key_env: OPENROUTER_API_KEY
+  best_value:
+    provider: openrouter
+    model: openai/gpt-5.6-terra
+    base_url: https://openrouter.ai/api/v1
+    api_key_env: OPENROUTER_API_KEY
+  paper_openai:
+    provider: openrouter
+    model: openai/gpt-4o-2024-08-06
+    base_url: https://openrouter.ai/api/v1
+    api_key_env: OPENROUTER_API_KEY
+  paper_claude:
+    provider: openrouter
+    model: anthropic/claude-3.5-sonnet
+    base_url: https://openrouter.ai/api/v1
+    api_key_env: OPENROUTER_API_KEY
+  paper_gemini:
+    provider: openrouter
+    model: google/gemini-pro-1.5
+    base_url: https://openrouter.ai/api/v1
+    api_key_env: OPENROUTER_API_KEY
+  fastuse_experimental:
+    provider: openrouter
+    model: google/gemini-3.1-pro-preview
+    base_url: https://openrouter.ai/api/v1
+    api_key_env: OPENROUTER_API_KEY

dataset/statistics/app_genre.csv

@@ -0,0 +1,101 @@
+id,genre
+1026760,Casual
+1033960,Casual
+1063530,Adventure
+1113370,Action
+1150310,Sports
+1157070,Casual
+1172280,Casual
+1178780,Strategy
+1210650,Adventure
+1217930,Indie
+1245640,Adventure
+1248270,Adventure
+1250210,Action
+1264160,Simulation
+1269890,Action
+1270910,Simulation
+1295570,Indie
+1298890,Adventure
+1334900,Indie
+1337060,Education
+1337530,Adventure
+1382350,Adventure
+1388030,Sports
+1394210,Action
+1394410,Action
+1404360,Adventure
+1453730,Casual
+1456280,Action
+1480650,Adventure
+1481400,Adventure
+1485260,Casual
+1486660,Adventure
+1488730,Sports
+1512840,Simulation
+1550280,Action
+1561560,Casual
+1595490,Action
+1601810,Adventure
+1612820,Casual
+1652800,Indie
+1676620,Adventure
+1695870,Indie
+1705030,Action
+1707840,Casual
+1730290,Simulation
+1801560,Action
+1805510,Simulation
+1825460,Simulation
+1862180,Simulation
+1906880,Action
+1931980,Indie
+1960620,Strategy
+2020950,Indie
+2025000,Simulation
+2057000,Simulation
+2077870,Action
+2089520,RPG
+2163140,Adventure
+2193270,Adventure
+2224020,Simulation
+2293180,Action
+2308940,Strategy
+269170,Sports
+343740,Indie
+438100,Massively Multiplayer
+451980,Simulation
+457380,Indie
+457550,Simulation
+463290,Casual
+490250,Simulation
+497820,Simulation
+513490,Indie
+518580,Indie
+528580,Adventure
+529150,Strategy
+533970,Design & Illustration
+591680,Simulation
+600140,Strategy
+605850,Casual
+622310,Casual
+631660,Strategy
+660520,Action
+667010,Action
+714100,Animation & Modeling
+716260,Action
+720300,Adventure
+726910,Casual
+731790,Simulation
+758210,Simulation
+790750,Action
+812460,Indie
+815280,Action
+866540,Adventure
+891960,Simulation
+898080,Indie
+910190,Action
+954160,Indie
+982710,Indie
+997760,Sports
+998660,Simulation

dataset/statistics/app_tag.csv

@@ -0,0 +1,101 @@
+id,tag
+1026760,Casual;VR;Immersive;Nature;Cinematic;Free to Play;Early Access;Colorful;3D;360 Video;Experimental;Singleplayer
+1033960,Casual;VR;Time Management;Funny;Family Friendly;Dog
+1063530,Adventure;Action;Indie;Casual;RPG;VR;Free to Play
+1113370,Action;Free to Play;Indie;Casual;Sports;VR;Sci-fi;Real-Time;Stylized;Futuristic;Competitive
+1150310,VR;3D Platformer;Skiing;Sports;Racing;Runner;Immersive Sim;Arcade;Walking Simulator;First-Person;3D;Cute;Time Manipulation;Casual;Cartoony;Colorful;Family Friendly;6DOF;Atmospheric;Nature
+1157070,Casual;Indie;VR;Puzzle;First-Person;Family Friendly;Relaxing;Singleplayer;Atmospheric;Solitaire;Physics;Tabletop
+1172280,Automobile Sim;VR;Experience;Cinematic;Psychedelic;Minimalist;6DOF;Transportation;Psychological Horror;Abstract;Comedy;Experimental;Atmospheric;Surreal;Horror;Casual;Singleplayer;Immersive Sim;Simulation;Indie
+1178780,Space Sim;Space;VR;Multiplayer;Co-op;Sci-fi;First-Person;RTS;Strategy;Singleplayer;Simulation;Real Time Tactics;PvP;Tactical;Action;War;Real-Time;Local Co-Op;Bullet Time;Real-Time with Pause
+1210650,Adventure;Free to Play;Indie;VR;Singleplayer;Escape Room
+1217930,VR;Flight;Multiplayer;Shooter;Simulation;Space Sim;FPS;Battle Royale;Action;Adventure;Massively Multiplayer;Indie;Early Access;Fantasy;Twin Stick Shooter;Futuristic;Robots;Sci-fi;Space;Surreal
+1245640,Interactive Fiction;VR;Story Rich;Narrative;6DOF;Historical;Education;Emotional;Documentary;First-Person;3D;Cinematic;Linear;Narration;Singleplayer;Feature Film;Adventure;Colorful;Casual;Exploration
+1248270,VR;Free to Play;Exploration;Interactive Fiction;Experience;Singleplayer;Casual;Surreal;Magic;Medieval;Adventure
+1250210,Indie;Action;Massively Multiplayer;Gore;Violent;Early Access;VR;Looter Shooter
+1264160,Immersive Sim;Simulation;Anime;Experience;VR;Comic Book;Cute;Dating Sim;Crowdfunded;FMV;Singleplayer;Realistic;Casual;Atmospheric;Drama;Conversation;Story Rich;Romance;Sexual Content;Funny
+1269890,Education;Arcade;Casual;Free to Play;On-Rails Shooter;Hack and Slash;VR;Action;Singleplayer;Indie;Strategy;RPG;Great Soundtrack
+1270910,Free to Play;Indie;Casual;Simulation;Villain Protagonist;Singleplayer
+1295570,Indie;VR;Story Rich;Atmospheric;Emotional
+1298890,Casual;Exploration;Walking Simulator;VR;Story Rich;Surreal;Atmospheric;Psychological;Adventure;Singleplayer;Narrative;Indie;Mystery
+1334900,Casual;Action;Arcade;VR;Parody;Hack and Slash;Level Editor;Singleplayer;6DOF;Gore;Violent;Indie
+1337060,Education;Science
+1337530,VR;Adventure;Puzzle;Stylized;Open World;Silent Protagonist;Nonlinear;Interactive Fiction;Story Rich;Fantasy;Indie;Narration;Physics;Atmospheric;Singleplayer;Cinematic;Exploration;6DOF;3D;First-Person
+1382350,Adventure;VR;Detective;Puzzle;Strategy;Singleplayer;Mystery;Investigation;Time Travel;Atmospheric;Interactive Fiction;First-Person;Indie;Dark;Supernatural;6DOF;Story Rich
+1388030,Sports;VR;PvP;Realistic;Competitive;Simulation;Physics;Turn-Based Tactics;Casual;Real Time Tactics;3D;Real-Time;Arcade;Funny;Action-Adventure;Multiplayer;Singleplayer;First-Person;Arena Shooter;Grand Strategy
+1394210,Action;VR;Arcade;Shooter;Shoot 'Em Up;Bullet Hell;3D;Aliens;Fantasy;Futuristic;Jet;Mechs;Robots;Sci-fi;Space;Bullet Time;Combat;Flight;PvE;Score Attack
+1394410,Action;VR;Magic;Swordplay;Fighting;PvP;Futuristic;Combat;PvE;Singleplayer;Multiplayer;Spectacle fighter;Sci-fi
+1404360,Casual;VR;Hidden Object;Investigation;Exploration;Collectathon;Cute;Relaxing;Atmospheric;Puzzle;Fantasy;Conversation;Narration;Family Friendly;Nature;Singleplayer;6DOF;Indie;Adventure;Free to Play
+1453730,Casual;Relaxing;Atmospheric;Experimental;Colorful;Psychedelic;Abstract;VR;Space;Free to Play;Family Friendly;Ambient;Early Access;3D;6DOF;Exploration;Education;Singleplayer
+1456280,Action;Fighting;Shooter;3D Fighter;Arena Shooter;VR;Flight;First-Person;Aliens;Cyberpunk;Post-apocalyptic;Sci-fi;PvP;Artificial Intelligence;Multiplayer;Singleplayer
+1480650,Adventure;Action-Adventure;Puzzle;VR;First-Person;Tactical;Snow;Choices Matter;Quick-Time Events;Action;Real Time Tactics;Magic;Fantasy;Survival;Lore-Rich;Arcade;Crafting;Physics;Superhero;Time Manipulation
+1481400,Horror;Lovecraftian;Atmospheric;Visual Novel;First-Person;Story Rich;Psychological Horror;Dark;Narration;Choose Your Own Adventure;VR;Adventure;Interactive Fiction;Point & Click;Mystery;Indie;Walking Simulator;Text-Based;Realistic;Emotional
+1485260,Relaxing;Casual;Abstract;Psychedelic;Experimental;Cinematic;3D Vision;Exploration;3D;Stylized;6DOF;Procedural Generation;Singleplayer;VR
+1486660,Casual;VR;Action;Dragons;Puzzle;Adventure;Exploration;Arcade;Walking Simulator;Education;3D;First-Person;Realistic;Dinosaurs;Logic;3D Vision;Narration;Action-Adventure;Singleplayer;Flight
+1488730,Casual;Simulation;Sports;eSports;Arcade;Immersive Sim;Grand Strategy;3D;VR;Real Time Tactics;Family Friendly;Nature;Physics;Singleplayer
+1512840,Casual;Simulation;Photo Editing;Design & Illustration;Utilities;VR;Experience;Walking Simulator
+1550280,VR;Atmospheric;Casual;Building;LEGO;Simulation;Interactive Fiction;Abstract;Family Friendly;Life Sim;Immersive Sim;Magic;Colorful;Physics;Cute;Sandbox;First-Person;Utilities;Moddable;Space
+1561560,Free to Play;VR;Singleplayer;Time Management;Puzzle;Sandbox;Physics;Casual;Management;Artificial Intelligence;Early Access;Simulation;Life Sim;Arcade;First-Person;Medieval;3D;Cute;Lovecraftian;Cartoony
+1595490,Action Roguelike;VR;Roguelite;Sailing;Perma Death;Arcade;Roguelike;First-Person;3D;Action;Post-apocalyptic;Casual;Cartoony;Dystopian;Management;6DOF;Artificial Intelligence;Resource Management;Singleplayer;Free to Play
+1601810,Adventure;Nature;VR
+1612820,Casual;VR;Funny;Cartoony;Hand-drawn;Arcade;Stylized;Free to Play;Singleplayer
+1652800,Puzzle;Casual;Female Protagonist;3D;First-Person;VR;Linear;Sci-fi;Indie;Science;Singleplayer
+1676620,Adventure;Education;Action-Adventure;Medical Sim;3D;VR;6DOF;Free to Play;Singleplayer
+1695870,Free to Play;VR;Walking Simulator;Singleplayer;Atmospheric;Surreal;Short;Colorful;Immersive;Linear;Stylized;Magic;6DOF;First-Person;Mystery;Relaxing;Exploration;Funny;Adventure;Indie
+1705030,Shooter;Sandbox;Platformer;VR;Multiplayer;Action;Parkour;eSports;Physics;PvP;Arena Shooter;Bullet Hell;FPS;3D;First-Person;Sports;Character Customization;Adventure;3D Platformer;Cartoony
+1707840,Casual;Strategy;Puzzle;Shooter;Match 3;VR;Cartoony;Colorful;Atmospheric;Nature;Relaxing;6DOF;Choices Matter;Singleplayer
+1730290,Casual;Simulation;Arcade;Tabletop;Visual Novel;VR;3D;Abstract;Artificial Intelligence;Free to Play;Sci-fi;Singleplayer
+1801560,Action;Shooter;Rhythm;Shoot 'Em Up;FPS;Bullet Hell;3D;Realistic;Destruction;Singleplayer;VR
+1805510,Action;Simulation;Action-Adventure;Life Sim;Exploration;Immersive Sim;3D;VR;America;Building;Capitalism;Comedy;Story Rich;Trading;Controller;Resource Management;Singleplayer;Indie;Arcade;Dark Humor
+1825460,Simulation;Education;VR;Immersive Sim;3D;Crafting;Realistic;Singleplayer;Hardware
+1862180,Simulation;VR;Space;Immersive Sim;Exploration;Education;Science;Realistic;Sci-fi;Physics;3D;Artificial Intelligence;Atmospheric;Space Sim;Open World;Singleplayer;Difficult;Early Access;Immersive;Golf
+1906880,Casual;Action;VR;Combat;Action-Adventure;First-Person;3D Platformer;FPS;Singleplayer;Hack and Slash;Free to Play;RPG;Indie;3D;War;Martial Arts;Atmospheric;PvE
+1931980,Escape Room;Local Co-Op;VR;Hand-drawn;Investigation;Co-op;Puzzle;Indie;Strategy
+1960620,Strategy;VR;Psychological Horror;Chess;Board Game;Tabletop;Singleplayer;Hentai
+2020950,VR;Emotional;6DOF;Immersive;Colorful;Cinematic;Atmospheric;Story Rich;Electronic Music;Visual Novel;Design & Illustration;Stylized;3D;Audio Production;Singleplayer;Drama;Abstract;First-Person;Casual;Linear
+2025000,VR;Arcade;Simulation;Roguelite;Funny;Management;Job Simulator;Perma Death;Psychedelic;Time Management;Immersive Sim;3D;Cooking;Casual;Physics;Action;Colorful;Indie;Nature;Singleplayer
+2057000,Casual;Simulation;Software;Puzzle;Utilities;Word Game;Immersive Sim;3D;3D Vision;Colorful;VR;First-Person;Education;Foreign;6DOF;Conversation;Open World;PvE;Singleplayer;Early Access
+2077870,Action;Adventure;Simulation;Strategy;Arcade;VR;First-Person;Atmospheric;PvE;Singleplayer;Choices Matter;Dark Humor;Free to Play;FPS;Funny;Futuristic;Memes;Sequel;Shoot 'Em Up;Multiple Endings
+2089520,VR;Massively Multiplayer;Anime;Multiplayer;Funny;3D Platformer;MMORPG;First-Person;Open World;6DOF;Nature;3D;Stylized;Atmospheric;Drama;Story Rich;Cartoon;Co-op;PvP;RPG
+2163140,VR;Puzzle;Singleplayer;Escape Room;Detective;Adventure;Investigation;Historical;Trains;Mystery;Walking Simulator;Exploration;Surreal;Stylized;Interactive Fiction;Hidden Object;Immersive Sim;Colorful;6DOF;Indie
+2193270,Adventure;VR;Historical;Sailing;Casual;Education;Visual Novel;Action-Adventure;Exploration;Immersive Sim;Colorful;First-Person;Stylized;Atmospheric;Diplomacy;Drama;Family Friendly;Political;Choices Matter;Linear
+2224020,Simulation;VR;Life Sim;Immersive Sim;Realistic;Singleplayer;Historical;Atmospheric;Free to Play;3D;Nudity;Walking Simulator;Nonlinear
+2293180,VR;Experimental;Shoot 'Em Up;Singleplayer;Action;Arcade;RPG;Shooter;Zombies;FPS;Combat;Physics;Free to Play;PvE
+2308940,VR;Building;Education;Strategy;LEGO;First-Person;Controller;Simulation;Arcade;Turn-Based Strategy;FPS;Real Time Tactics;Time Management;Colorful;Family Friendly;Logic;Modern;Tactical;6DOF;Base Building
+269170,VR;Snooker;Sports;Simulation;Casual;Pool;Indie;Local Multiplayer;Strategy;Physics;Multiplayer;Addictive
+343740,VR;Free to Play;Indie;Action;Education;Casual;Sci-fi;Singleplayer;Science;Exploration;3D Vision;Dynamic Narration;Funny;Comedy;Racing;Physics;Simulation;Space
+438100,VR;Free to Play;Multiplayer;Memes;Anime;Funny;Massively Multiplayer;First-Person;Early Access;Open World;Simulation;Casual;Adventure;MMORPG;Comedy;Sandbox;Action;Dating Sim;Survival Horror;Atmospheric
+451980,Simulation;Casual;VR;Education;Great Soundtrack;Exploration;First-Person;Physics;Free to Play;Indie;Atmospheric
+457380,Adventure;Simulation;Indie;Casual;VR;Episodic;Survival
+457550,VR;Free to Play;Software;Open World;Utilities;Split Screen;Multiplayer;3D;Movie;First-Person;Choose Your Own Adventure;Casual;Cinematic;Exploration;Simulation;Futuristic;Nature;Relaxing;Sci-fi;Conversation
+463290,VR;Indie;Casual;Immersive;Atmospheric
+490250,Simulation;Indie;Casual;Free to Play;VR;Trains;Relaxing;On-Rails Shooter
+497820,Free to Play;VR;Cyberpunk
+513490,Indie;VR;Historical;Free to Play;Education;World War II
+518580,Indie;VR;Adventure;Comedy;Free to Play;Dark Humor;Horror;Funny;Singleplayer
+528580,Adventure;Free to Play;VR;Early Access;Indie;Casual;Fantasy;Cinematic;First-Person;Dragons;Interactive Fiction;Experimental
+529150,Strategy;Indie;VR;Arcade;RTS;Free to Play
+533970,Design & Illustration;Free to Play;VR;Animation & Modeling
+591680,Simulation;Indie;VR;Physics;Casual;Funny;Addictive;Free to Play;Arcade;Management;FPS;Cartoony;3D;Colorful;Comedy;Dark Humor;Education;Relaxing;Score Attack;Singleplayer
+600140,Free to Play;VR;Strategy;Casual;Indie;Action;Puzzle;Gore;Violent;Escape Room;Sexual Content
+605850,Indie;Casual;VR;Pinball
+622310,Strategy;Free to Play;Massively Multiplayer;Casual;Simulation;VR
+631660,Free to Play;Strategy;Casual;Sports;Tabletop;VR;Chess
+660520,Simulation;Free to Play;Action;Indie;VR;Mechs;Robots
+667010,Action;Indie;Casual;Violent;VR;Cartoony;Dark Comedy;Zombies;Post-apocalyptic;Survival Horror;Vampire;Funny;First-Person;Comedy;Shooter;Blood;Dark Humor;Horror;Sports;1980s
+714100,Animation & Modeling;VR;Anime;Sexual Content;Nudity
+716260,Free to Play;Adventure;Action;Indie;VR;Shooter;Western
+720300,Indie;Adventure;Free to Play;VR;Short;Singleplayer;Sci-fi;Atmospheric;Futuristic;Puzzle;Space;Great Soundtrack;Relaxing;Beautiful;Experimental
+726910,Free to Play;VR;Robots;Adventure;Sci-fi;Casual;Short;First-Person;Multiplayer;Singleplayer;Survival;Strategy;Action;Atmospheric;Open World;Indie;Interactive Fiction;Immersive Sim;Local Multiplayer;Nudity
+731790,Casual;Simulation;VR;Indie;Competitive;Atmospheric;Singleplayer;Violent;Difficult;Horror;Fast-Paced;First-Person;Funny;War;Action;Tactical;Comedy;Replay Value;Colorful;Arcade
+758210,Adventure;Early Access;Simulation;VR;Escape Room
+790750,Arena Shooter;Hero Shooter;FPS;Archery;PvP;Competitive;Shooter;Arcade;Character Customization;Silent Protagonist;First-Person;VR;Action;Casual;Dystopian;Robots;Co-op;Multiplayer;Online Co-Op;Singleplayer
+812460,Indie;Casual;Simulation;World War II;Historical;VR
+815280,Strategy;Action;Indie;Early Access;VR;Pirates
+866540,Free to Play;Adventure;Indie;RPG;VR;Short;Funny;Sci-fi;Physics;Space;Singleplayer;Story Rich;Atmospheric;Comedy;First-Person;Robots
+891960,Free to Play;Casual;Simulation;VR;Space Sim;Space;Exploration;Physics
+898080,Indie;Sexual Content;VR;Free to Play;Psychological Horror
+910190,Free to Play;Indie;Action;VR;Sci-fi;FPS;Aliens;Space;Arcade
+954160,Free to Play;Indie;VR
+982710,Free to Play;Indie;VR;Local Multiplayer;Local Co-Op;Minigames;Villain Protagonist
+997760,Sports;VR;Arcade;Baseball
+998660,Free to Play;Early Access;VR;Action;Simulation;Casual;Flight;Multiplayer

dataset/statistics/plot.py

@@ -0,0 +1,179 @@
+import json
+import pandas as pd
+from collections import defaultdict
+from matplotlib import pyplot as plt
+import numpy as np
+
+SPLIT_GENRE = {
+        "TRAIN": [
+            "Casual",
+            "Adventure",
+            "Action",
+            "Indie"
+        ],
+        "VAL": [
+            "Strategy",
+            "Education",
+            "RPG",
+            "Massively Multiplayer",
+            "Design & Illustration",
+            "Animation & Modeling"
+        ],
+        "TEST": [
+            "Simulation",
+            "Sports"
+        ]
+    }
+
+
+def parse_img_id(img_id):
+    img_id = str(img_id)
+    return int(img_id[:-3]), int(img_id[-3:])
+
+
+def plot(data, img_name, is_genre=False):
+    plt.rcParams.update({'font.size': 18})
+    colors = []
+    for cat in data.keys():
+        if cat in SPLIT_GENRE['TRAIN']:
+            colors.append('tab:blue')
+        elif cat in SPLIT_GENRE['VAL']:
+            colors.append('tab:orange')
+        elif cat in SPLIT_GENRE['TEST']:
+            colors.append('tab:green')
+        else:
+            colors.append('gray')
+    
+    
+    if is_genre:
+        plt.figure(figsize=(10, 6))
+        plt.grid(True, axis='y', zorder=0)
+        plt.bar(data.keys(), data.values(), color=colors, zorder=3)
+        plt.xticks(rotation=30, ha='right')
+        plt.legend(handles=[
+            plt.Line2D([0], [0], color='tab:blue', lw=10, label='Train'),
+            plt.Line2D([0], [0], color='tab:orange', lw=10, label='Val'),
+            plt.Line2D([0], [0], color='tab:green', lw=10, label='Test'),
+        ])
+    else:
+        plt.figure(figsize=(15, 6))
+        plt.grid(True, axis='y', zorder=0)
+        plt.bar(data.keys(), data.values(), zorder=3)
+        plt.xticks(rotation=45, ha='right')
+    plt.gca().yaxis.set_major_locator(plt.MaxNLocator(integer=True))
+    for i, v in enumerate(data.values()):
+        plt.text(i, v, str(v), ha='center', va='bottom')
+    y_max = max(data.values())
+    plt.ylim(0, 1.1 * y_max)
+    
+    plt.tight_layout()
+    plt.savefig(img_name + '.png')
+    plt.savefig(img_name + '.pdf')
+
+
+def plot_genre():
+
+    df = pd.read_csv('app_genre.csv')
+    with open('../data/coco_merged/annotations/semantics.json', 'r') as f:
+        dataset = json.load(f)
+
+    app_img_map = {}
+
+    for img in dataset['images']:
+        img_id = img['id']
+        app_id, _ = parse_img_id(img['id'])
+        if app_id not in app_img_map:
+            app_img_map[app_id] = []
+        app_img_map[app_id].append(img_id)
+
+    gnr_app_map = defaultdict(list)
+
+    for i in range(len(df)):
+        app_id = df['id'][i]
+        tags = df['genre'][i].split(';')
+        for tag in tags:
+            gnr_app_map[tag].append(int(app_id))
+
+
+    gnr_app_count = {cat: len(apps) for cat, apps in gnr_app_map.items()}
+    gnr_img_count = {cat: sum([len(app_img_map[app]) for app in apps]) for cat, apps in gnr_app_map.items()}
+    gnr_anno_count = {cat: sum([len([anno for anno in dataset['annotations'] if anno['image_id'] in app_img_map[app]]) for app in apps]) for cat, apps in gnr_app_map.items()}
+
+    split_genre_order = {genre: i for i, genre in enumerate(SPLIT_GENRE['TRAIN'] + SPLIT_GENRE['VAL'] + SPLIT_GENRE['TEST'])}
+    gnr_app_count = {cat: gnr_app_count[cat] for cat in sorted(gnr_app_count, key=lambda x: split_genre_order.get(x, float('inf')))}
+    gnr_img_count = {cat: gnr_img_count[cat] for cat in sorted(gnr_img_count, key=lambda x: split_genre_order.get(x, float('inf')))}
+    gnr_anno_count = {cat: gnr_anno_count[cat] for cat in sorted(gnr_anno_count, key=lambda x: split_genre_order.get(x, float('inf')))}
+    
+    
+    plot(gnr_app_count, 'genre_app_count', is_genre=True)
+    plot(gnr_img_count, 'genre_img_count', is_genre=True)
+    plot(gnr_anno_count, 'genre_anno_count', is_genre=True)
+    
+
+def plot_cat():
+
+    df = pd.read_csv('app_tag.csv')
+    with open('../data/coco_merged/annotations/semantics.json', 'r') as f:
+        dataset = json.load(f)
+
+    app_img_map = {}
+
+    for img in dataset['images']:
+        img_id = img['id']
+        app_id, _ = parse_img_id(img['id'])
+        if app_id not in app_img_map:
+            app_img_map[app_id] = []
+        app_img_map[app_id].append(img_id)
+
+    cat_app_map = defaultdict(list)
+
+    for i in range(len(df)):
+        app_id = df['id'][i]
+        tags = df['tag'][i].split(';')
+        for tag in tags:
+            cat_app_map[tag].append(int(app_id))
+
+    cat_app_map.pop('VR')
+
+    cat_app_count = {cat: len(apps) for cat, apps in cat_app_map.items()}
+    cat_img_count = {cat: sum([len(app_img_map[app]) for app in apps]) for cat, apps in cat_app_map.items()}
+    cat_anno_count = {cat: sum([len([anno for anno in dataset['annotations'] if anno['image_id'] in app_img_map[app]]) for app in apps]) for cat, apps in cat_app_map.items()}
+
+    cat_app_count = {cat: count for cat, count in list(cat_app_count.items())[:30]}
+    cat_img_count = {cat: count for cat, count in list(cat_img_count.items())[:30]}
+    cat_anno_count = {cat: count for cat, count in list(cat_anno_count.items())[:30]}
+
+    cat_app_count = {cat: count for cat, count in sorted(cat_app_count.items(), key=lambda item: item[1], reverse=True)}
+    cat_img_count = {cat: count for cat, count in sorted(cat_img_count.items(), key=lambda item: item[1], reverse=True)}
+    cat_anno_count = {cat: count for cat, count in sorted(cat_anno_count.items(), key=lambda item: item[1], reverse=True)}
+
+    plot(cat_app_count, 'tag_app_count')
+    plot(cat_img_count, 'tag_img_count')
+    plot(cat_anno_count, 'tag_anno_count')
+
+
+def plot_cat_anno():
+    with open('../data/coco_merged/annotations/semantics.json', 'r') as f:
+        dataset = json.load(f)
+    
+    cat_catname_map = {}
+    for cat in dataset['categories']:
+        cat_catname_map[cat['id']] = cat['name']
+        
+    cat_anno_map = defaultdict(list)
+    for anno in dataset['annotations']:
+        cat_anno_map[cat_catname_map[anno['category_id']]].append(anno)
+    
+    print(len(cat_anno_map['button']))
+    cat_anno_map.pop('button')
+    
+    cat_anno_count = {cat: len(annos) for cat, annos in cat_anno_map.items()}
+    cat_anno_count = {cat: count for cat, count in sorted(cat_anno_count.items(), key=lambda item: item[1], reverse=True)}
+    cat_anno_count = {cat: count for cat, count in list(cat_anno_count.items())[:30]}
+    plot(cat_anno_count, 'ige_cat_anno_count')
+    
+
+
+plot_genre()
+plot_cat()
+plot_cat_anno()

docs/ASSETS.md

@@ -0,0 +1,59 @@
+# Required external assets
+
+This public copy includes the APE-L\(_D\) checkpoint required by the main detector. It intentionally excludes generated artifacts, credential-bearing files, private data, historical experiment outputs, and full dataset image payloads.
+
+## Minimum smoke test
+
+To validate one Orienter inference example, provide:
+
+| Asset | Expected location or interface | Required metadata |
+| --- | --- | --- |
+| One representative XR screenshot | a test-only folder matching the question manifest | image license/permission and expected image ID |
+| Matching question/context fixture | JSONL consumed by `approach/run_vlm.py`, generated by `scripts/generate_questions.py` when screenshots are available | selected prompt, provider, model ID, filename convention, and expected schema |
+| App metadata cache | an external path supplied through `--app-metadata-cache` | `app_id`, app name, app description, source URL, version/date, license/permission |
+| APE-L\(_D\) checkpoint | bundled at `approach/ovod/APE/ape_d_model_final.pth` | official source, Apache-2.0 license, SHA-256, and size recorded in `docs/MODEL_MANIFEST.md` |
+| Built APE extension | local build/install under the target CUDA environment | generated from source after cloning; compiled artifacts are intentionally not redistributed |
+| One supported provider credential | `openrouter-run` or an equivalent process-scoped secret manager | access to the selected OpenRouter model and an explicit spending limit |
+| Expected prediction | private comparison fixture | expected candidate labels and approximate boxes |
+
+Run `python -B scripts/verify_assets.py --repo-root .` to validate the bundled checkpoint size. Add `--hash-checkpoint` for SHA-256 verification. When screenshots and metadata are mounted, pass `--questions`, `--images-dir`, and `--app-metadata-cache` to check image naming, image ID consistency, and metadata coverage. Pass `--embedding-cache /path/to/embedding_dict.json` to hash the external semantic cache against `evaluation/cache_manifest.json` before an offline evaluation.
+
+The default Stage 2 runner also verifies the released checkpoint hash before loading it. Loading any different PyTorch checkpoint requires the explicit `--trust-custom-checkpoint` flag and should be done only after independent provenance and checksum verification.
+
+## Full paper reproduction
+
+Provide or publish all of the following as versioned assets:
+
+1. The 1,552-image XR UI screenshot payload and a canonical versioned annotation release. Selected COCO-format evaluation ground-truth splits are included in this repository, but must be tied to the public dataset version and license.
+2. App, genre, and context-sensitive split manifests, including the extra non-interactable context annotations.
+3. Steam/application metadata used for global context, preferably as a pinned JSON/JSONL cache that avoids live-page drift. The public runner fails closed on missing cache entries when `--app-metadata-cache` is supplied.
+4. The complete prompt templates, demonstration examples, and the random-selection/seed policy.
+5. Exact model identifiers and access paths for every Orienter variant evaluated in the paper. The code defaults to OpenRouter `openai/gpt-5.6-sol`; exact paper profiles are included for GPT-4o-2024-08-06, Claude 3.5 Sonnet, and Gemini 1.5 Pro. If a provider retires a historical route, supply equivalent access through a custom profile or direct vendor credentials and document the substitution.
+6. The bundled APE-L\(_D\) checkpoint identity, official source, revision, license, and checksum in `docs/MODEL_MANIFEST.md`.
+7. A working CUDA/PyTorch/Detectron2 APE environment with the APE native extension built locally from the released source tree.
+8. The reviewed `embedding-3` semantic cache identified by `evaluation/cache_manifest.json`. Mount it read-only and use offline mode for historical reproduction. Zhipu access is optional and should be enabled only to extend a separate cache copy when a new model produces an uncached category.
+9. Expected per-split metrics or prediction checksums for a deterministic regression check.
+10. Hardware, CUDA, PyTorch, Python, and package versions from the successful experiment environment. See `docs/ENVIRONMENT.md` for the current public build boundary.
+
+## Credential names found in code
+
+The following names are referenced by the main or optional baseline paths:
+
+- `OPENAI_API_KEY`
+- `OPENROUTER_API_KEY`
+- `GOOGLE_API_KEY`
+- `ANTHROPIC_API_KEY`
+- `ZHIPU_API_KEY`
+- `ARK_API_KEY`
+- `DASHSCOPE_API_KEY`
+- `DEEPSEEK_API_KEY`
+- `THEB_API_KEY`
+- `INTERNVL_API_KEY`
+
+Only `OPENROUTER_API_KEY` is needed for the recommended Orienter inference path; the other variables belong to optional evaluation or baseline code. Never send credentials through Git, issue trackers, logs, or README examples. Rotate any credential that has previously appeared in a file or experiment log.
+
+Stage 1 sends full screenshots to the configured remote provider; reflection additionally sends crops and annotated scenes. Do not submit private or unauthorized screenshots. Review the provider's data-handling terms and keep generated candidates, predictions, traces, metadata caches, and error files outside the public repository because they can reveal source-image content or filenames.
+
+## Publication format
+
+For each data/model asset, publish: human-readable name, stable URL, version/revision, SHA-256, license, expected local path, compressed/uncompressed size, and a minimal schema example. The bundled APE checkpoint should be published through Hugging Face/LFS-style large-file storage; keep private raw data and generated experiment outputs outside the public repository.

docs/ENVIRONMENT.md

@@ -0,0 +1,93 @@
+# Environment and APE build
+
+Orienter has two runtime layers. The lightweight manifest, provider, and evaluation utilities use the root `requirements.txt`. Stage 2 additionally needs PyTorch, CUDA, Detectron2, detrex, and the compiled `ape._C` extension.
+
+## Public starting point
+
+Use Python 3.10 or later for the public environment. This is required by the current Pillow security release; the historical Python 3.9 environment is recorded below for provenance, not recommended for processing untrusted images.
+
+For lightweight utilities:
+
+```bash
+python -B -m venv .venv
+source .venv/bin/activate
+python -B -m pip install --upgrade pip
+python -B -m pip install -r requirements.txt
+```
+
+For the CUDA detector, `environment.reference.yml` is a compatibility starting point based on the verified experiment host, with Python and Pillow moved to currently supported public versions:
+
+```bash
+conda env create -f environment.reference.yml
+conda activate orienter-ape-reference
+bash scripts/build_ape_extension.sh
+```
+
+The environment file pins PyTorch 2.2.0 with CUDA 12.1 and the exact Detectron2 and detrex commits observed in the working server environment. It deliberately builds the vendored APE source after environment creation instead of redistributing a machine-specific binary. It is not a cross-platform lockfile or a security attestation for the older research stack. Audit the resolved environment on the publication host. The target machine needs a CUDA toolkit compatible with its PyTorch build, a supported C++ compiler, and enough memory/disk to compile the extension.
+
+`scripts/build_ape_extension.sh` does not install or upgrade dependencies. It performs an editable build of the vendored APE source in the active environment, then runs the environment checker. Review the active environment before executing it.
+
+## Verify the runtime
+
+From the repository root:
+
+```bash
+python -B scripts/check_environment.py
+python -B scripts/verify_assets.py --repo-root . --hash-checkpoint
+python -B scripts/smoke_control_path.py
+python scripts/run_tests.py
+```
+
+The environment check fails if the detector dependency stack is unavailable. `--allow-no-cuda` relaxes only the CUDA-availability check, and `--skip-ape-extension` relaxes only the compiled `ape._C` check; both remain diagnostic flags and still require PyTorch, torchvision, Detectron2, and detrex. They do not turn this command into a checker for the lightweight root requirements, and they are not valid evidence for Stage 2 inference readiness.
+
+After mounting screenshots and application metadata, validate their contracts before a live run:
+
+```bash
+python -B scripts/verify_assets.py \
+  --repo-root . \
+  --questions /absolute/path/to/questions.jsonl \
+  --images-dir /absolute/path/to/images \
+  --app-metadata-cache /absolute/path/to/app_metadata.json
+```
+
+To reproduce semantic evaluation offline, separately mount the frozen cache and verify its full SHA-256:
+
+```bash
+python -B scripts/verify_assets.py \
+  --repo-root . \
+  --skip-checkpoint \
+  --embedding-cache /absolute/path/to/embedding_dict.json
+```
+
+## Verified historical server environment
+
+The release checkpoint, one-image APE smoke test, and test suite were exercised in the historical server `conda` environment named `ape` with:
+
+| Component | Observed version |
+| --- | --- |
+| Python | 3.9.18 |
+| PyTorch | 2.2.0+cu121 |
+| torchvision | 0.17.0+cu121 |
+| CUDA runtime reported by PyTorch | 12.1 |
+| CUDA toolkit (`nvcc`) | 12.2 |
+| NVIDIA driver | 535.261.03 |
+| GPU | NVIDIA A100-PCIE-40GB |
+| GCC | 11.4.0 |
+| Detectron2 | 0.6, commit `017abbfa5f2c2a2afa045200c2af9ccf2fc6227f` |
+| detrex | 0.3.0, commit `776058ec229be37a5ff2a2b0bb54133bdd5da663` |
+| transformers | 4.37.2 |
+| NumPy | 1.22.4 |
+| SciPy | 1.7.3 |
+| Pillow | 10.2.0 |
+| OpenCV Python | 4.9.0.80 |
+| xformers | 0.0.24 |
+| fairscale | 0.4.13 |
+| lvis | 0.5.3 |
+
+`torch.cuda.is_available()` returned `True`, and `ape._C` imported successfully when invoked from the Orienter repository root. Running Python from inside `approach/ovod/APE` before building the local extension can shadow the installed APE package with the unbuilt source tree; build first and run the public entrypoints from the Orienter root.
+
+The historical environment includes an old Pillow version and is retained only as execution provenance. Do not use it for arbitrary or untrusted images. A fresh build of `environment.reference.yml` on the final Hugging Face target remains part of the release gate because compiled CUDA extensions are host-sensitive.
+
+## Files that must stay local
+
+Do not commit compiled extensions, `build/`, `*.egg-info`, model caches, virtual/conda environments, logs, generated candidates or predictions, or configuration files containing credentials or private absolute paths. Re-run the release hygiene tests after every live validation.

docs/MODEL_MANIFEST.md

@@ -0,0 +1,11 @@
+# Model manifest
+
+This public release bundles only the model file required by the main Orienter detector. Baseline training checkpoints, fold-specific weights, duplicate backups, and prior experiment outputs are excluded.
+
+| File | Size | SHA-256 | Purpose | License |
+| --- | ---: | --- | --- | --- |
+| `approach/ovod/APE/ape_d_model_final.pth` | 5,956,547,279 bytes | `3548f41a3238148180e08fd4b16c71f4abc3ac3caf9c8434444462d1bdb7f965` | APE-L\(_D\) detector checkpoint used by Stage 2 grounding | Apache-2.0 |
+
+Upstream source: [official `shenyunhang/APE` checkpoint](https://huggingface.co/shenyunhang/APE/blob/6ee15cf54c3930528ac49be165b45d5bcb1f4fd9/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl_20230829_162438/model_final.pth), pinned at repository revision `6ee15cf54c3930528ac49be165b45d5bcb1f4fd9`. The official file reports the same SHA-256 and the upstream model repository declares Apache-2.0. The corresponding license text is retained at `approach/ovod/APE/LICENSE`.
+
+The checkpoint uses PyTorch pickle serialization. Load it only through the pinned APE environment and treat replacement checkpoint files as executable untrusted input until independently verified.