Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/sample.mp4 filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+YOLO-World/demo/sample_images/bus.jpg filter=lfs diff=lfs merge=lfs -text
+YOLO-World/demo/sample_images/zidane.jpg filter=lfs diff=lfs merge=lfs -text
+YOLO-World/third_party/mmyolo/demo/demo.jpg filter=lfs diff=lfs merge=lfs -text
+YOLO-World/third_party/mmyolo/demo/dog.jpg filter=lfs diff=lfs merge=lfs -text
+YOLO-World/third_party/mmyolo/demo/large_image.jpg filter=lfs diff=lfs merge=lfs -text
+YOLO-World/third_party/mmyolo/resources/qq_group_qrcode.jpg filter=lfs diff=lfs merge=lfs -text
+YOLO-World/third_party/mmyolo/resources/zhihu_qrcode.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+output
+pretrained

Dockerfile

@@ -0,0 +1,17 @@
+# basic dockerfile
+FROM python:3.10
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+
+# install deps
+RUN pip install torch==2.4.1 torchvision==0.19.1 torchaudio==2.4.1 wheel
+RUN pip install -r requirements.txt
+
+COPY --chown=user . /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,12 +1,15 @@
 ---
 title: TSTAR
-emoji: 🌖
-colorFrom: green
-colorTo: gray
-sdk: gradio
-sdk_version: 5.20.1
-app_file: app.py
+emoji: ⚡️🔍🎥🎥🎥🎥🔍⚡️
+colorFrom: yellow
+colorTo: indigo
+sdk: docker
 pinned: false
+app_port: 7860
 ---
 
+<!-- sdk: gradio
+sdk_version: 5.9.1
+app_file: app.py -->
+
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

TStar/TStarFramework.py

@@ -0,0 +1,347 @@
+
+"""
+TStarSearcher: Comprehensive Video Frame Search Tool
+
+This script allows searching for specific objects within a video using YOLO object detection and GPT-4 for question-answering. It leverages the TStar framework's universal Grounder, YOLO interface, and video searcher to identify relevant frames and answer questions based on the detected objects.
+
+Usage:
+    python tstar_searcher.py --video_path path/to/video.mp4 --question "Your question here" --options "A) Option1\nB) Option2\nC) Option3\nD) Option4"
+"""
+
+import os
+import sys
+import cv2
+import torch
+import copy
+import logging
+import argparse
+import numpy as np
+import matplotlib.pyplot as plt
+from dataclasses import dataclass, field
+from typing import List, Optional, Tuple
+from decord import VideoReader, cpu
+from scipy.interpolate import UnivariateSpline
+
+# Import custom TStar interfaces
+from TStar.interface_llm import TStarUniversalGrounder
+from TStar.interface_owl import OWLInterface
+from TStar.interface_searcher import TStarSearcher
+from TStar.utils import save_as_gif
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s [%(levelname)s] %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout)
+    ]
+)
+logger = logging.getLogger(__name__)
+
+
+class TStarFramework:
+    """
+    Main class for performing object-based frame search and question-answering in a video.
+    """
+
+    def __init__(
+        self,
+        video_path: str,
+        heuristic_scorer: OWLInterface,
+        grounder: TStarUniversalGrounder,
+        question: str,
+        options: str,
+        search_nframes: int = 8,
+        grid_rows: int = 4,
+        grid_cols: int = 4,
+        output_dir: str = './output',
+        confidence_threshold: float = 0.6,
+        search_budget: int = 1000,
+        prefix: str = 'stitched_image',
+        config_path: Optional[str] = None,
+        checkpoint_path: Optional[str] = None,
+        device: str = "cuda:0"
+    ):
+        """
+        Initialize VideoSearcher.
+
+        Args:
+            video_path (str): Path to the input video file.
+            yolo_scorer (YoloV5Interface): YOLO interface instance.
+            grounder (TStarUniversalGrounder): Universal Grounder instance.
+            question (str): The question for question-answering.
+            options (str): Multiple-choice options for the question.
+            search_nframes (int, optional): Number of top frames to return. Default is 8.
+            grid_rows (int, optional): Number of rows in the image grid. Default is 4.
+            grid_cols (int, optional): Number of columns in the image grid. Default is 4.
+            output_dir (str, optional): Directory to save outputs. Default is './output'.
+            confidence_threshold (float, optional): YOLO detection confidence threshold. Default is 0.6.
+            search_budget (int, optional): Maximum number of frames to process during search. Default is 1000.
+            prefix (str, optional): Prefix for output filenames. Default is 'stitched_image'.
+            config_path (str, optional): Path to the YOLO configuration file. Default is None.
+            checkpoint_path (str, optional): Path to the YOLO model checkpoint. Default is None.
+            device (str, optional): Device for model inference (e.g., "cuda:0" or "cpu"). Default is "cuda:0".
+        """
+        self.video_path = video_path
+        self.yolo_scorer = heuristic_scorer
+        self.grounder = grounder
+        self.question = question
+        self.options = options
+        self.search_nframes = search_nframes
+        self.grid_rows = grid_rows
+        self.grid_cols = grid_cols
+        self.output_dir = output_dir
+        self.confidence_threshold = confidence_threshold
+        self.search_budget = search_budget
+        self.prefix = prefix
+        self.config_path = config_path
+        self.checkpoint_path = checkpoint_path
+        self.device = device
+
+        # Ensure the output directory exists
+        os.makedirs(self.output_dir, exist_ok=True)
+        logger.info("VideoSearcher initialized successfully.")
+
+        self.results = {}
+
+    def run(self):
+        """
+        Execute the complete video search and question-answering process.
+        """
+        # Use Grounder to get target and cue objects
+        target_objects, cue_objects = self.get_grounded_objects()
+
+        # Initialize TStarSearcher
+        video_searcher = TStarSearcher(
+            video_path=self.video_path,
+            target_objects=target_objects,
+            cue_objects=cue_objects,
+            search_nframes=self.search_nframes,
+            image_grid_shape=(self.grid_rows, self.grid_cols),
+            output_dir=self.output_dir,
+            confidence_threshold=self.confidence_threshold,
+            search_budget=self.search_budget,
+            prefix=self.prefix,
+            owl=self.yolo_scorer
+        )
+        
+        logger.info(f"TStarSearcher initialized successfully for video {self.video_path}.")
+
+        # Perform search
+        all_frames, time_stamps = self.perform_search(video_searcher)
+
+        # Save retrieved frames
+        self.save_frames(all_frames, time_stamps)
+        self.save_searching_iters(video_searcher)
+        # Plot and save score distribution
+        self.plot_and_save_scores(video_searcher)
+
+        # Perform question-answering on retrieved frames
+        answer = self.perform_qa(all_frames)
+        print("QA Answer:", answer)
+
+        logger.info("VideoSearcher completed successfully.")
+
+    def get_grounded_objects(self) -> Tuple[List[str], List[str]]:
+        """
+        Use Grounder to obtain target and cue objects.
+
+        Returns:
+            Tuple[List[str], List[str]]: Lists of target objects and cue objects.
+        """
+        # Example code; should be implemented based on Grounder's interface
+        # For example:
+        target_objects, cue_objects = self.grounder.inference_query_grounding(
+            video_path=self.video_path,
+            question=self.question
+        )
+        # Here, assuming fixed target and cue objects
+        # target_objects = ["couch"]  # Target objects to find
+        # cue_objects = ["TV", "chair"]  # Cue objects
+
+        logger.info(f"Target objects: {target_objects}")
+        logger.info(f"Cue objects: {cue_objects}")
+        self.results["Searching_Objects"] = {"target_objects": target_objects, "cue_objects": cue_objects}
+        return target_objects, cue_objects
+
+    def perform_search(self, video_searcher: TStarSearcher) -> Tuple[List[np.ndarray], List[float]]:
+        """
+        Execute the frame search process and retrieve relevant frames and timestamps.
+
+        Args:
+            video_searcher (TStarSearcher): Instance of TStarSearcher.
+
+        Returns:
+            Tuple[List[np.ndarray], List[float]]: List of frames and their corresponding timestamps.
+        """
+        all_frames, time_stamps = video_searcher.search_with_visualization()
+        logger.info(f"Found {len(all_frames)} frames, timestamps: {time_stamps}")
+        
+        self.results['timestamps'] = time_stamps
+        return all_frames, time_stamps
+
+    def perform_qa(self, frames: List[np.ndarray]) -> str:
+        """
+        Perform question-answering on the retrieved frames.
+
+        Args:
+            frames (List[np.ndarray]): List of frames to analyze.
+
+        Returns:
+            str: Answer generated by VLM.
+        """
+        answer = self.grounder.inference_qa(
+            frames=frames,
+            question=self.question,
+            options=self.options
+        )
+        self.results['answer'] = answer
+        return answer
+
+    def plot_and_save_scores(self, video_searcher: TStarSearcher):
+        """
+        Plot the score distribution and save the plot.
+
+        Args:
+            video_searcher (TStarSearcher): Instance of TStarSearcher.
+        """
+        plot_path = os.path.join(self.output_dir, "score_distribution.png")
+        video_searcher.plot_score_distribution(save_path=plot_path)
+        logger.info(f"Score distribution plot saved to {plot_path}")
+
+    def save_frames(self, frames: List[np.ndarray], timestamps: List[float]):
+        """
+        Save the retrieved frames as image files.
+
+        Args:
+            frames (List[np.ndarray]): List of frames to save.
+            timestamps (List[float]): Corresponding timestamps of the frames.
+        """
+        for idx, (frame, timestamp) in enumerate(zip(frames, timestamps)):
+            frame_path = os.path.join(
+                self.output_dir,
+                f"frame_{idx}_at_{timestamp:.2f}s.jpg"
+            )
+            cv2.imwrite(frame_path, cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
+            logger.info(f"Saved frame to {frame_path}")
+
+    def save_searching_iters(self, video_searcher, video_ids=[]):
+        # # 定义 resize 操作，目标大小为 (640, 640)
+        # resize_transform = T.Resize((1024, 1024))
+        # resized_frames_tensor = resize_transform(resized_frames_tensor)
+        
+        image_grid_iters = video_searcher.image_grid_iters # iters, b, image # b = 1 for v1
+        detect_annotot_iters = video_searcher.detect_annotot_iters # iters, b, image
+        detect_bbox_iters = video_searcher.detect_bbox_iters #iters, b, n_objects, xxyy, 
+            
+        fps = 1  # 设置帧率为 2
+        for b in range(len(image_grid_iters[0])):
+            images =  [image_grid_iter[b] for image_grid_iter in image_grid_iters]
+            anno_images = [detect_annotot_iter[b] for detect_annotot_iter in detect_annotot_iters] 
+
+            frame_size = (anno_images[0].shape[1], anno_images[0].shape[0])  # 获取图像大小 (宽度, 高度)
+
+            # 设置视频的参数
+            video_id=self.video_path.split("/")[-1].split(".")[0]
+            output_video_path = os.path.join(self.output_dir, f"{video_id}.gif")  # 视频保存路径
+            save_as_gif(images=anno_images, output_gif_path=output_video_path)
+
+    
+
+
+def initialize_TStar_Scorer(
+    heuristic: str,
+    device: str
+) -> OWLInterface:
+    """
+    Initialize the YOLO object detection model.
+
+    Args:
+        config_path (str): Path to the YOLO configuration file.
+        checkpoint_path (str): Path to the YOLO model checkpoint.
+        device (str): Device for model inference (e.g., "cuda:0").
+
+    Returns:
+        YoloWorldInterface: Initialized YOLO interface instance.
+
+    Raises:
+        FileNotFoundError: If the configuration file or checkpoint file is not found.
+    """
+
+    model_choice = 'owl_model'
+    if model_choice == 'owl_model':
+        model_name="google/owlvit-base-patch32"
+        owl_interface = OWLInterface(
+            config_path = model_name,
+            checkpoint_path=None,
+            device="cuda:0"
+        )
+    logger.info("YoloWorldInterface initialized successfully.")
+    return owl_interface
+
+
+def run_tstar(
+    video_path: str,
+    question: str,
+    options: str,
+    grounder: str,
+    heuristic: str,
+    openai_api_key: str,
+    device: str = "cuda:0",
+    search_nframes: int = 8,
+    grid_rows: int = 4,
+    grid_cols: int = 4,
+    confidence_threshold: float = 0.6,
+    search_budget: float = 0.5,
+    output_dir: str = './output',
+):
+    """
+    Executes the TStar video frame search and QA process.
+    
+    Args:
+        video_path (str): Path to the input video file.
+        question (str): Question for video content QA.
+        options (str): Multiple-choice options for the question.
+        config_path (str): Path to the YOLO configuration file.
+        checkpoint_path (str): Path to the YOLO model checkpoint.
+        device (str): Device for model inference (e.g., "cuda:0" or "cpu").
+        search_nframes (int): Number of top frames to return.
+        grid_rows (int): Number of rows in the image grid.
+        grid_cols (int): Number of columns in the image grid.
+        confidence_threshold (float): YOLO detection confidence threshold.
+        search_budget (float): Maximum ratio of frames to process during search.
+        output_dir (str): Directory to save outputs.
+        prefix (str): Prefix for output filenames.
+    
+    Returns:
+        dict: Results containing detected objects, timestamps, and the QA answer.
+    """
+    # Initialize Grounder and YOLO
+    grounder = TStarUniversalGrounder(backend="gpt4", model_name="gpt-4o", gpt4_api_key=openai_api_key)
+    
+    TStar_Scorer = initialize_TStar_Scorer(
+        heuristic=heuristic,
+        device=device
+    )
+
+    # Initialize and run the search framework
+    searcher = TStarFramework(
+        grounder=grounder,
+        heuristic_scorer=TStar_Scorer,
+        video_path=video_path,
+        question=question,
+        options=options,
+        search_nframes=search_nframes,
+        grid_rows=grid_rows,
+        grid_cols=grid_cols,
+        output_dir=output_dir,
+        confidence_threshold=confidence_threshold,
+        search_budget=search_budget,
+        device=device
+    )
+    searcher.run()
+
+    return {
+        "Grounding Objects": searcher.results.get('Searching_Objects', []),
+        "Frame Timestamps": searcher.results.get('timestamps', []),
+        "Answer": searcher.results.get('answer', "No answer generated")
+    }

TStar/interface_llm.py

@@ -0,0 +1,451 @@
+
+import torch
+import os
+from tqdm import tqdm
+from typing import Dict, Optional, Sequence, List
+import transformers
+import re
+
+import openai
+from typing import List, Dict
+from PIL import Image
+import base64
+import io
+try:
+    import cv2
+except ImportError:
+    cv2 = None
+    print("Warning: OpenCV is not installed, video frame extraction will not work.")
+from TStar.utils import *
+
+class LlavaInterface:
+    """
+    示例：封装对 Llava 模型的推理调用。
+    关键在于对外暴露统一的方法 inference(query, frames, **kwargs)。
+    """
+    def __init__(self, model_path: str, model_base: Optional[str] = None):
+        # 这里是加载 Llava 模型等的逻辑
+        # self.tokenizer, self.model = ...
+        self.model_path = model_path
+        self.model_base = model_base
+        print(f"[LlavaInterface] model_path={model_path}, model_base={model_base}")
+
+    def inference(
+        self,
+        query: str,
+        frames: Optional[List[Image.Image]] = None,
+        system_message: str = "You are a helpful assistant.",
+        temperature: float = 0.2,
+        top_p: Optional[float] = None,
+        num_beams: int = 1,
+        max_tokens: int = 512,
+        **kwargs
+    ) -> str:
+        """
+        对外暴露统一的推理接口。
+
+        query: 用户输入，可能包含文本+<image>标记
+        frames: 对应的图像帧列表
+        system_message: 系统提示
+        其它参数根据需要自行添加
+        """
+        
+        # 模拟推理逻辑，需要你自己实现
+        print("[LlavaInterface] Inference called with query:", query)
+        print("[LlavaInterface] frames count:", len(frames) if frames else 0)
+
+        # 真实场景下，你会调用 Llava 模型进行推理
+        return "Fake Response from LlavaInterface"
+    
+class GPT4Interface:
+    def __init__(self,model="gpt-4o", api_key=None):
+        """
+        Initialize the GPT-4 API client.
+
+        Reads the OpenAI API key from the environment variable `OPENAI_API_KEY`.
+        """
+        self.api_key = api_key
+        self.model_name = model
+        if api_key==None:
+            self.api_key = os.getenv("OPENAI_API_KEY")
+        if not self.api_key:
+            raise ValueError("Environment variable OPENAI_API_KEY is not set.")
+        openai.api_key = self.api_key
+
+    def inference_text_only(self, query: str, system_message: str = "You are a helpful assistant.", temperature: float = 0.7, max_tokens: int = 1000) -> str:
+        """
+        Perform inference using the GPT-4 API.
+
+        Args:
+            query (str): User's query or input.
+            system_message (str): System message to guide the model's behavior.
+            temperature (float): Sampling temperature for the response.
+            max_tokens (int): Maximum number of tokens for the response.
+
+        Returns:
+            str: The response generated by the GPT-4 model.
+        """
+        messages = [
+            {"role": "system", "content": system_message},
+            {"role": "user", "content": query},
+        ]
+
+        try:
+            response = openai.chat.completions.create(
+                model=self.model_name,
+                messages=messages,
+                temperature=temperature,
+                max_tokens=max_tokens,
+            )
+            
+            return response.choices[0].message.content.strip()
+        except Exception as e:
+            return f"Error: {str(e)}"
+    
+    def inference_with_frames(self, query: str, frames: List[Image.Image], system_message: str = "You are a helpful assistant.", temperature: float = 0.7, max_tokens: int = 1000) -> str:
+        """
+        Perform inference using the GPT-4 API with video frames as context.
+
+        Args:
+            query (str): User's query or input.
+            frames (List[Image.Image]): List of PIL.Image objects to provide visual context.
+            system_message (str): System message to guide the model's behavior.
+            temperature (float): Sampling temperature for the response.
+            max_tokens (int): Maximum number of tokens for the response.
+
+        Returns:
+            str: The response generated by the GPT-4 model.
+        """
+
+        # Messages format
+        inputs = [{"type": "text", "text": query}]
+
+        # Encode frames as Base64 strings
+        for i, frame in enumerate(frames):
+            try:
+                # Convert PIL Image to Base64 string
+                frame_base64 = encode_image_to_base64(frame)
+                visual_context = {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{frame_base64}",
+                            "detail": "low"
+                            }
+                    }
+                # Adding visual context (images) to messages if supported by the model
+                inputs.append(visual_context)
+
+            except Exception as e:
+                return f"Error encoding frame {i}: {str(e)}"
+
+        messages = [
+            {"role": "system", "content": system_message},
+            {"role": "user", "content": inputs},
+        ]
+        try:
+            response = openai.chat.completions.create(
+                model=self.model_name,
+                messages=messages,
+                temperature=temperature,
+                max_tokens=max_tokens
+            )
+            return response.choices[0].message.content.strip()
+        except Exception as e:
+            return f"Error: {str(e)}"
+
+    
+
+    def inference_qa(self, question: str, options: str, frames: List[Image.Image] = None, system_message: str = "You are a helpful assistant.", temperature: float = 0.7, max_tokens: int = 500) -> str:
+        """
+        Perform inference for a multiple-choice question with optional visual frames as context.
+
+        Args:
+            question (str): The question to answer.
+            options (str): Multiple-choice options formatted as a string.
+            frames (List[Image.Image], optional): List of PIL.Image objects to provide additional visual context.
+            system_message (str): System message to guide the model's behavior.
+            temperature (float): Sampling temperature for the response.
+            max_tokens (int): Maximum number of tokens for the response.
+
+        Returns:
+            str: The selected option or answer.
+        """
+        # Construct query
+        query = f"Question: {question}\nOptions: {options}\nAnswer with the letter corresponding to the best choice."
+
+        # Messages format
+        inputs = [{"type": "text", "text": query}]
+
+        if frames:
+            # Encode frames as Base64 strings
+            for i, frame in enumerate(frames):
+                try:
+                    frame_base64 = encode_image_to_base64(frame)
+                    visual_context = {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{frame_base64}",
+                            "detail": "low"
+                        }
+                    }
+                    # Adding visual context (images) to messages if supported by the model
+                    inputs.append(visual_context)
+
+                except Exception as e:
+                    return f"Error encoding frame {i}: {str(e)}"
+
+        messages = [
+            {"role": "system", "content": system_message},
+            {"role": "user", "content": inputs},
+        ]
+
+        try:
+            response = openai.chat.completions.create(
+                model="gpt-4o",
+                messages=messages,
+                temperature=temperature,
+                max_tokens=max_tokens
+            )
+            return response.choices[0].message.content.strip()
+        except Exception as e:
+            return f"Error: {str(e)}"
+
+    def inference_with_frames_all_in_one(self, query: str, frames: List[Image.Image], system_message: str = "You are a helpful assistant.", temperature: float = 0.7, max_tokens: int = 1000) -> str:
+        """
+        Perform inference using the GPT-4 API with video frames as context.
+        Args:
+            query (str): User's query or input. image tag: <image>
+            frames (List[Image.Image]): List of PIL.Image objects to provide visual context.
+            system_message (str): System message to guide the model's behavior.
+            temperature (float): Sampling temperature for the response.
+            max_tokens (int): Maximum number of tokens for the response.
+        Returns:
+            str: The response generated by the GPT-4 model.
+        """
+        # Split query by <image>
+        parts = query.split("<image>")
+        inputs = []
+
+        # Add text and images alternately to inputs
+        for i, part in enumerate(parts):
+            if part.strip():
+                inputs.append({"type": "text", "text": part.strip()})
+            if i < len(frames):  # Ensure we don't exceed the number of available frames
+                try:
+                    frame_base64 = encode_image_to_base64(frames[i])
+                    visual_context = {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/jpeg;base64,{frame_base64}",
+                            "detail": "low"
+                        }
+                    }
+                    inputs.append(visual_context)
+                except Exception as e:
+                    return f"Error encoding frame {i}: {str(e)}"
+
+        messages = [
+            {"role": "system", "content": system_message},
+            {"role": "user", "content": inputs},
+        ]
+
+        try:
+            response = openai.chat.completions.create(
+                model=self.model_name,
+                messages=messages,
+                temperature=temperature,
+                max_tokens=max_tokens
+            )
+            return response.choices[0].message.content.strip()
+        except Exception as e:
+            return f"Error: {str(e)}"
+
+class TStarUniversalGrounder:
+    """
+    结合了原先 TStarGrounder 与 TStarGPTGrounder 的功能，
+    可以通过 backend 参数切换到底层使用的是 LlavaInterface 还是 GPT4Interface。
+    """
+    def __init__(
+        self,
+        backend: str = "gpt",
+        model_name: str = "gpt-4o",
+        model_path: Optional[str] = None,
+        model_base: Optional[str] = None,
+        gpt4_api_key: Optional[str] = None,
+        num_frames: Optional[int] = 8,
+    ):
+        """
+        backend: "llava" 或 "gpt4"
+        model_path, model_base: Llava 模型的路径及版本
+        gpt4_model_name, gpt4_api_key: GPT4 的模型名称及 API Key
+        """
+        self.backend = backend.lower()
+        self.num_frames = num_frames
+        if self.backend == "llava":
+            # 初始化 LlavaInterface
+            if not model_path:
+                raise ValueError("Please provide model_path for LlavaInterface")
+            self.VLM_model_interfance = LlavaInterface(model_path=model_path, model_base=model_base)
+        elif self.backend == "gpt4":
+            # 初始化 GPT4Interface
+            self.VLM_model_interfance = GPT4Interface(model=model_name, api_key=gpt4_api_key)
+        else:
+            raise ValueError("backend must be either 'llava' or 'gpt4'.")
+
+    def inference_query_grounding(
+        self,
+        video_path: str,
+        question: str,
+        options: Optional[str] = None,
+        temperature: float = 0.0,
+        max_tokens: int = 512
+    ) -> Dict[str, List[str]]:
+        """
+        识别可作为答案依据的 target_objects 和可能辅助判断的 cue_objects。
+        """
+
+        frames = load_video_frames(video_path=video_path, num_frames=self.num_frames)
+        # 构建 prompt
+        system_prompt = (
+            "Here is a video:\n"
+            + "\n".join(["<image>"] * len(frames))  
+            + "\nHere is a question about the video:\n"
+            f"Question: {question}\n"
+        )
+        if options:
+            system_prompt += f"Options: {options}\n"
+        system_prompt += (
+            "\nWhen answering this question about the video:\n"
+            "1. What key objects to locate the answer?\n"
+            "   - List potential key objects (short sentences, separated by commas).\n"
+            "2. What cue objects might be near the key objects and might appear in the scenes?\n"
+            "   - List potential cue objects (short sentences, separated by commas).\n\n"
+            "Please provide your answer in two lines, directly listing the key and cue objects, separated by commas."
+        )
+
+        # 统一走 self.interface.inference # need more abstract function
+        response = self.VLM_model_interfance.inference_with_frames_all_in_one(
+            query=system_prompt,
+            frames=frames,
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+
+        # 根据预期格式解析响应
+        lines = response.split("\n")
+        if len(lines) < 2:
+            # print(response)
+            raise ValueError(f"Unexpected response format from inference_query_grounding() --> {response}.")
+
+        target_objects = [self.check_objects_str(obj) for obj in lines[0].split(",") if obj.strip()]
+        cue_objects = [self.check_objects_str(obj) for obj in lines[1].split(",") if obj.strip()]
+        
+        return target_objects, cue_objects
+    def check_objects_str(self, obj: str):
+        obj = obj.lower() #小写
+        obj = obj.strip().replace("1. ", "")
+        obj = obj.strip().replace("2. ", "") 
+        obj = obj.strip().replace(".", "")
+        obj = obj.strip().replace("key objects: ", "")
+        obj = obj.strip().replace("cue objects: ", "")
+        obj = obj.strip().replace(": ", "")
+
+        return obj
+
+
+
+
+    def inference_qa(
+        self,
+        frames: List[Image.Image],
+        question: str,
+        options: str,
+        temperature: float = 0.2,
+        max_tokens: int = 128
+    ) -> str:
+        """
+        多选推理，返回最可能的选项（如 A、B、C、D）。
+        """
+        system_prompt = (
+            "Select the best answer to the following multiple-choice question based on the video.\n"
+            + "\n".join(["<image>"] * len(frames))  
+            + f"\nQuestion: {question}\n"
+            + f"Options: {options}\n\n"
+            "Answer with the option’s letter from the given choices directly."
+        )
+
+        response = self.VLM_model_interfance.inference_with_frames_all_in_one(
+            query=system_prompt,
+            frames=frames,
+            temperature=temperature,
+            max_tokens=30
+        )
+        return response.strip()
+
+    def inference_openend_qa(
+        self,
+        frames: List[Image.Image],
+        question: str,
+        # options: str,
+        temperature: float = 0.2,
+        max_tokens: int = 2048
+    ) -> str:
+        """
+        多选推���，返回最可能的选项（如 A、B、C、D）。
+        """
+        system_prompt = (
+            "Answer with the question in short based on the video.\n"
+            + "\n".join(["<image>"] * len(frames))  
+            + f"\nQuestion: {question}\n"
+        )
+
+        response = self.VLM_model_interfance.inference_with_frames_all_in_one(
+            query=system_prompt,
+            frames=frames,
+            temperature=temperature,
+            max_tokens=30
+        )
+        return response.strip()
+
+if __name__ == "__main__":
+    """
+    测试示例。
+    """
+    # 1) 使用 Llava 作为底层模型
+    # print("=== Using Llava backend ===")
+    # llava_grounder = TStarUniversalGrounder(
+    #     backend="llava",
+    #     model_path="/path/to/llava",
+    #     model_base="v1.0"
+    # )
+    frames_fake = [ # 随记噪声更好
+        Image.open("./output_image.jpg"),
+        Image.open("/home/yejinhui/Projects/VisualSearch/output_image.jpg")
+    ]
+    # result_grounding_llava = llava_grounder.inference_query_grounding(
+    #     frames=frames_fake,
+    #     question="What objects are in the video?",
+    # )
+    # print("Llava Grounding Result:", result_grounding_llava)
+
+    # 2) 使用 GPT-4 作为底层模型
+    print("\n=== Using GPT-4 backend ===")
+    gpt4_grounder = TStarUniversalGrounder(
+        backend="gpt4",
+        model_name="gpt-4o",
+        gpt4_api_key=None
+    )
+    searchable_objects = gpt4_grounder.inference_query_grounding(
+        frames=frames_fake,
+        question="What objects are in the video?"
+    )
+    print("GPT-4 Grounding Result:", searchable_objects)
+
+    # 3) 多选问答示例
+    question_mc = "How many cats can be seen?\n"
+    options_mc = "A) 0\nB) 1\nC) 2\nD) 3\n"
+    # answer_llava = llava_grounder.inference_qa(frames_fake, question_mc, options_mc)
+    # print("Llava QA Answer:", answer_llava)
+
+    answer_gpt4 = gpt4_grounder.inference_qa(frames_fake, question_mc, options_mc)
+    print("GPT-4 QA Answer:", answer_gpt4)

TStar/interface_owl.py

@@ -0,0 +1,136 @@
+from typing import List
+from transformers import OwlViTProcessor, OwlViTForObjectDetection
+import cv2
+import numpy as np
+from PIL import Image
+import torch
+import supervision as sv
+from torch.cuda.amp import autocast
+class owlInterface:
+    def __init__(self):
+        """
+        Initialize the YOLO-World model with the given configuration and checkpoint.
+
+        Args:
+        """
+        
+     
+        pass
+
+class OWLInterface(owlInterface):
+    # def __init__(self, model_name="google/owlvit-base-patch32"):
+    def __init__(self, config_path: str, checkpoint_path: None, device: str = "cuda:0"):
+        self.processor, self.model = self.load_model_and_tokenizer(config_path)
+        self.device = device
+        self.model = self.model.to(self.device)
+        self.texts = ["couch", "table", "woman"]
+
+    def load_model_and_tokenizer(self, model_name):
+        processor = OwlViTProcessor.from_pretrained(model_name)
+        model = OwlViTForObjectDetection.from_pretrained(model_name)
+        return processor, model
+
+    def forward_model(self, inputs):
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+        return outputs
+
+    def inference(self, image_path, use_amp: bool = False):
+        with Image.open(image_path).convert("RGB") as image:
+            width, height = image.size
+        inputs = self.processor(text=self.texts, images=image, return_tensors="pt").to(self.device)
+
+        # Run model inference
+        outputs = self.forward_model(inputs)
+
+        # Post-process outputs
+        target_size = torch.tensor([[height, width]])
+        results = self.processor.post_process_grounded_object_detection(
+            outputs=outputs, target_sizes=target_size)[0]
+        detections = sv.Detections.from_transformers(transformers_results=results)
+        return detections
+    
+    def inference_detector(self, images, use_amp: bool = False):
+        # batch_images = []
+        # for i in range(4): #@Jinhui why there are hard code and why four? bug!!
+        #     for j in range(4):
+        #         # Extract the smaller image from the grid
+        #         small_image = image[i*120:(i+1)*120, j*160:(j+1)*160]
+        #         batch_images.append(small_image)
+        # # batch_images = np.array(batch_images)
+        # images = [Image.fromarray(np.uint8(img)).convert("RGB") for img in batch_images]
+        batch_images = np.array(images)
+        inputs = self.processor(text= self.texts, images=batch_images[0], return_tensors="pt").to(self.device)
+        height, width = batch_images[0].shape[:2]
+        detections_inbatch = []
+        with torch.no_grad():
+            # Run model inference 
+            outputs = self.forward_model(inputs)
+
+            target_sizes = torch.tensor([[height, width] for i in batch_images]) 
+            results = self.processor.post_process_grounded_object_detection(
+                outputs=outputs, target_sizes=target_sizes, threshold=0.05)
+        for result in results:
+            detections = sv.Detections.from_transformers(transformers_results=result)
+            detections_inbatch.append(detections)
+
+        
+        check = True
+        if check:
+            # save first image for checking
+            bounding_box_annotator = sv.BoxAnnotator()
+            annotated_image = bounding_box_annotator.annotate(batch_images[0] , detections_inbatch[0])
+
+            output_image = Image.fromarray(annotated_image[:, :, ::-1])
+            output_image.save("./annotated_image.png")
+            self.detections_inbatch = detections_inbatch
+        return detections_inbatch
+
+    def bbox_visualization(self, images, detections_inbatch):
+        # image =  Image.open(image_path).convert("RGB")
+        # output_image.save(output_path)
+        # detections = self.inference(images)
+        # Annotate image
+        # detections = self.inference(images)
+        bounding_box_annotator = sv.BoxAnnotator()
+        annotated_images = []
+        for image, detections in zip(images,detections_inbatch):
+            annotated_image = bounding_box_annotator.annotate(image, detections)
+            # output_image = Image.fromarray(annotated_image[:, :, ::-1])
+            annotated_images.append(annotated_image)
+            
+        return annotated_images
+    def reparameterize_object_list(self, target_objects: List[str], cue_objects: List[str]):
+        """
+        Reparameterize the detect object list to be used by the OWL model.
+
+        Args:
+            target_objects (List[str]): List of target object names.
+            cue_objects (List[str]): List of cue object names.
+        """
+        # Combine target objects and cue objects into the final text format
+        combined_texts = target_objects + cue_objects
+
+        # Format the text prompts for the YOLO model
+        self.texts = [[obj.strip()] for obj in combined_texts] + [[' ']]
+
+        # Reparameterize the YOLO model with the provided text prompts
+        # self.model.reparameterize(self.texts)
+        
+def main():
+    model_choice = 'owl_model'
+    
+    image_path = "/home/anabella/projects/MLLM/TSTAR/data/score/annotated_image.png"    
+    output_path = "/home/anabella/projects/MLLM/TSTAR/data/score/annotated_image3.png"
+
+    if model_choice == 'owl_model':
+        model_name="google/owlvit-base-patch32"
+        owl_interface = OWLInterface(
+            config_path = model_name,
+            checkpoint_path=None,
+            device="cuda:0"
+        )
+        owl_interface.bbox_visualization(image_path, output_path)
+
+if __name__ == "__main__":
+    main()

TStar/interface_searcher.py

@@ -0,0 +1,704 @@
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+from dataclasses import dataclass, field
+from typing import List, Optional, Tuple
+from decord import VideoReader, cpu
+from scipy.interpolate import UnivariateSpline
+import copy
+from tqdm import tqdm
+import os
+import sys
+import cv2
+import copy
+import logging
+
+# Assuming YoloWorldInterface is defined elsewhere and imported correctly
+# from your_project.yolo_interface import YoloWorldInterface
+# 导入自定义的 TStar 接口
+# from TStar.interface_yolo import YoloWorldInterface, YoloV5Interface, YoloInterface
+from .interface_owl import OWLInterface, owlInterface
+class TStarSearcher:
+    """
+    A class to perform keyframe search in a video using object detection and dynamic sampling.
+
+    Attributes:
+        video_path (str): Path to the video file.
+        target_objects (List[str]): List of target objects to find.
+        cue_objects (List[str]): List of cue objects for context.
+        confidence_threshold (float): Minimum confidence threshold for object detection.
+        search_nframes (int): Number of keyframes to search for.
+        image_grid_shape (Tuple[int, int]): Shape of the image grid for detection.
+        output_dir (Optional[str]): Directory to save outputs.
+        profix (str): Prefix for output files.
+        object2weight (dict): Weights assigned to specific objects.
+        raw_fps (float): Original frames per second of the video.
+        total_frame_num (int): Total number of frames adjusted for sampling rate.
+        duration (float): Duration of the video in seconds.
+        remaining_targets (List[str]): Targets yet to be found.
+        search_budget (int): Budget for the number of frames to process.
+        score_distribution (np.ndarray): Scores assigned to each frame.
+        P_history (List[List[float]]): History of probability distributions.
+        non_visiting_frames (np.ndarray): Indicator for frames not yet visited.
+        yolo (YoloWorldInterface): YOLO interface for object detection.
+    """
+
+    def __init__(
+        self,
+        video_path: str,
+        target_objects: List[str],
+        cue_objects: List[str],
+        search_nframes: int = 8,
+        image_grid_shape: Tuple[int, int] = (8, 8),
+        search_budget: float = 0.1,
+        output_dir: Optional[str] = None,
+        prefix: str = None,
+        confidence_threshold: float = 0.5,
+        object2weight: Optional[dict] = None,
+        model_choice: str ="owl",
+        owl = None,
+    ):
+        """
+        Initializes the TStarSearcher object with video properties and configurations.
+        
+        Args:
+            video_path (str): Path to the input video file.
+            target_objects (List[str]): List of objects to detect as primary targets.
+            cue_objects (List[str]): List of contextual objects to aid detection.
+            cue_object (Optional[str]): A single cue object for additional focus.
+            search_nframes (int): Number of keyframes to identify.
+            image_grid_shape (Tuple[int, int]): Grid dimensions for image tiling.
+            output_dir (Optional[str]): Directory to store results.
+            profix (str): Prefix for saved output files.
+            confidence_threshold (float): Threshold for object detection confidence.
+            object2weight (Optional[dict]): Mapping of objects to their respective detection weights.
+            config_path (str): Path to the YOLO configuration file.
+            checkpoint_path (str): Path to the YOLO model checkpoint.
+            device (str): Device for model inference (e.g., "cuda:0").
+        """
+        self.video_path = video_path
+        self.target_objects = target_objects
+        self.cue_objects = cue_objects
+        self.search_nframes = search_nframes
+        self.image_grid_shape = image_grid_shape
+        self.output_dir = output_dir
+        self.profix = prefix
+        self.confidence_threshold = confidence_threshold
+        self.object2weight = object2weight if object2weight else {}
+        self.fps = 1  # Sampling at 1 fps # TODO look at this
+        self.model_choice = model_choice
+        # Video properties
+        cap = cv2.VideoCapture(self.video_path)
+        if not cap.isOpened():
+            raise ValueError(f"Cannot open video file: {self.video_path}")
+        self.raw_fps = cap.get(cv2.CAP_PROP_FPS)
+        self.total_frame_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        self.duration = self.total_frame_num / self.raw_fps
+
+        # Adjust total frame number based on sampling rate
+        self.total_frame_num = int(self.duration * self.fps)
+        self.remaining_targets = target_objects.copy()
+        self.search_budget = min(1000, self.total_frame_num*search_budget)
+
+        # Initialize distributions
+        self.score_distribution = np.zeros(self.total_frame_num)
+        self.P_history = []
+        self.non_visiting_frames = np.ones(self.total_frame_num)
+        self.P = np.ones(self.total_frame_num) * self.confidence_threshold * 0.3
+
+        # Initialize YOLO interface TODO : allow for YOLOV5
+        self.owl = None
+        # self.reset_yolo_vocabulary(target_objects=target_objects, cue_objects=cue_objects)
+        for object in target_objects:
+            self.object2weight[object] = 1.0
+        for object in cue_objects:
+            self.object2weight[object] = 0.5
+        #TODO: put in if statement
+        # Initialize OWL interface
+        model_name="google/owlvit-base-patch32"
+        self.owl = owl
+        # self.owl = OWLInterface(
+        #     config_path = model_name,
+        #     checkpoint_path=None,
+        #     device="cuda:0"
+        # )
+    
+        # self.reset_owl_vocabulary(target_objects=target_objects, cue_objects=cue_objects)
+        for object in target_objects:
+            self.object2weight[object] = 1.0
+        for object in cue_objects:
+            self.object2weight[object] = 0.5
+
+
+    def reset_yolo_vocabulary(self, target_objects: List[str], cue_objects: List[str]):
+        """
+        Dynamically resets the YOLO vocabulary with the specified target and cue objects.
+
+        Args:
+            target_objects (List[str]): New list of target objects for detection.
+            cue_objects (List[str]): New list of cue objects for detection context.
+        """
+        self.target_objects = target_objects
+        self.cue_objects = cue_objects
+        self.owl.reparameterize_object_list(target_objects, cue_objects)
+
+    ### --- Detection Methods --- ###
+
+    def imageGridScoreFunction(
+        self,
+        images: List[np.ndarray],
+        output_dir: Optional[str],
+        image_grids: Tuple[int, int]
+    ) -> Tuple[np.ndarray, List[List[List[str]]]]:
+        """
+        Perform object detection on a batch of images using the YOLO interface.
+
+        Args:
+            images (List[np.ndarray]): List of images to process.
+            output_dir (Optional[str]): Directory to save detection results.
+            image_grids (Tuple[int, int]): Dimensions of the image grid (rows, cols).
+
+        Returns:
+            Tuple[np.ndarray, List[List[List[str]]]]: Confidence maps and detected object lists.
+                - confidence_maps: numpy array of shape (num_images, grid_rows, grid_cols)
+                - detected_objects_maps: list of lists, each sublist corresponds to a grid_image and contains detected objects per cell
+        """
+        if len(images) == 0:
+            return np.array([]), []
+
+        grid_rows, grid_cols = image_grids
+        grid_height = images[0].shape[0] / grid_rows
+        grid_width = images[0].shape[1] / grid_cols
+
+        confidence_maps = []
+        detected_objects_maps = []
+
+        # Perform detection on all images  --AI
+        for image in images:
+            if self.model_choice == "yolo":
+                # Run the YOLO inference
+                detections = self.owl.inference_detector(
+                    images=[image],  # Single image as a batch
+                    max_dets=50,
+                    use_amp=False
+                )
+            elif self.model_choice == "owl":
+                # Run the OWL inference
+                detections = self.owl.inference_detector(
+                    images=[image],  # Single image as a batch 
+                    use_amp=False
+                )
+
+            # Initialize confidence map and detected objects map
+            confidence_map = np.zeros((grid_rows, grid_cols))
+            detected_objects_map = [[] for _ in range(grid_rows * grid_cols)]
+
+            # Process detections
+            for detection in detections:
+                for bbox, label, confidence in zip(detection.xyxy, detection.class_id, detection.confidence):
+                    # Convert class ID to object name
+                    if self.model_choice == "yolo":
+                        object_name = self.owl.texts[label][0] #@Jinhui TBD for YOLOWorld
+                    elif self.model_choice == "owl":
+                        object_name = self.owl.texts[label][0]
+
+                    # Apply object weight if available
+                    weight = self.object2weight.get(object_name, 0.5)
+                    adjusted_confidence = confidence * weight
+
+                    # Calculate bounding box center
+                    x_min, y_min, x_max, y_max = bbox
+                    box_center_x = (x_min + x_max) / 2
+                    box_center_y = (y_min + y_max) / 2
+
+                    # Map center to grid cell
+                    grid_x = int(box_center_x // grid_width)
+                    grid_y = int(box_center_y // grid_height)
+
+                    # Ensure grid indices are valid
+                    grid_x = min(grid_x, grid_cols - 1)
+                    grid_y = min(grid_y, grid_rows - 1)
+
+                    # Update confidence map and detected objects
+                    cell_index = grid_y * grid_cols + grid_x
+                    confidence_map[grid_y, grid_x] = max(confidence_map[grid_y, grid_x], adjusted_confidence)
+                    detected_objects_map[cell_index].append(object_name)
+
+            confidence_maps.append(confidence_map)
+            detected_objects_maps.append(detected_objects_map)
+
+        return np.stack(confidence_maps), detected_objects_maps
+
+    def read_frame_batch(self, video_path: str, frame_indices: List[int]) -> Tuple[List[int], np.ndarray]:
+        """
+        Reads a batch of frames from the video at specified indices.
+
+        Args:
+            video_path (str): Path to the video file.
+            frame_indices (List[int]): Indices of frames to read.
+
+        Returns:
+            Tuple[List[int], np.ndarray]: List of indices and corresponding frame array.
+        """
+        vr = VideoReader(video_path, ctx=cpu(0))
+        return frame_indices, vr.get_batch(frame_indices).asnumpy()
+
+    def create_image_grid(self, frames: List[np.ndarray], rows: int, cols: int) -> np.ndarray:
+        """
+        Combine frames into a single image grid.
+
+        Args:
+            frames (List[np.ndarray]): List of frame images.
+            rows (int): Number of rows in the grid.
+            cols (int): Number of columns in the grid.
+
+        Returns:
+            np.ndarray: Combined image grid.
+        """
+        if len(frames) != rows * cols:
+            raise ValueError("Frame count does not match grid dimensions")
+
+        # Resize frames to fit the grid
+        resized_frames = [cv2.resize(frame, (160, 120)) for frame in frames]  # Resize to 160x120
+        grid_rows = [np.hstack(resized_frames[i * cols:(i + 1) * cols]) for i in range(rows)]
+        return np.vstack(grid_rows)
+
+    ### --- Scoring Methods --- ###
+
+    def score_image_grids(
+        self,
+        images: List[np.ndarray],
+        image_grids: Tuple[int, int]
+    ) -> Tuple[np.ndarray, List[List[List[str]]]]:
+        """
+        Generate confidence maps and detected objects for each image grid.
+
+        Args:
+            images (List[np.ndarray]): List of image grids to detect objects.
+            image_grids (Tuple[int, int]): Grid dimensions (rows, cols).
+
+        Returns:
+            Tuple[np.ndarray, List[List[List[str]]]]: Confidence maps and detected objects maps.
+        """
+        return self.imageGridScoreFunction(
+            images=images,
+            output_dir=self.output_dir,
+            image_grids=image_grids
+        )
+
+
+    def store_score_distribution(self):
+        """
+        Stores a copy of the current probability distribution to the history.
+        """
+        self.P_history.append(copy.deepcopy(self.P).tolist())
+
+    def update_top_25_with_window(
+        self,
+        frame_confidences: List[float],
+        sampled_frame_indices: List[int],
+        window_size: int = 5
+    ):
+        """
+        Update score distribution for top 25% frames and their neighbors.
+
+        Args:
+            frame_confidences (List[float]): Confidence scores for sampled frames.
+            sampled_frame_indices (List[int]): Corresponding frame indices.
+            window_size (int): Number of neighboring frames to update.
+        """
+        # Calculate the threshold for top 25%
+        top_25_threshold = np.percentile(frame_confidences, 75)
+
+        # Identify top 25% frames
+        top_25_indices = [
+            frame_idx for frame_idx, confidence in zip(sampled_frame_indices, frame_confidences)
+            if confidence >= top_25_threshold
+        ]
+
+        # Update neighboring frames
+        for frame_idx in top_25_indices:
+            for offset in range(-window_size, window_size + 1):
+                neighbor_idx = frame_idx + offset
+                if 0 <= neighbor_idx < len(self.score_distribution):
+                    self.score_distribution[neighbor_idx] = max(
+                        self.score_distribution[neighbor_idx],
+                        self.score_distribution[frame_idx]/(abs(offset) + 1) 
+                    )
+
+    def spline_keyframe_distribution(
+        self,
+        non_visiting_frames: np.ndarray,
+        score_distribution: np.ndarray,
+        video_length: int
+    ) -> np.ndarray:
+        """
+        Generate a probability distribution over frames using spline interpolation.
+
+        Args:
+            non_visiting_frames (np.ndarray): Indicator array for frames not yet visited.
+            score_distribution (np.ndarray): Current score distribution over frames.
+            video_length (int): Total number of frames.
+
+        Returns:
+            np.ndarray: Normalized probability distribution over frames.
+        """
+        # Extract indices and scores of visited frames
+        frame_indices = np.array([idx for idx, visited in enumerate(non_visiting_frames) if visited == 0])
+        observed_scores = np.array([score_distribution[idx] for idx in frame_indices])
+
+        # If no frames have been visited, return uniform distribution
+        if len(frame_indices) == 0:
+            return np.ones(video_length) / video_length
+
+        # Spline interpolation
+        spline = UnivariateSpline(frame_indices, observed_scores, s=0.5)
+        all_frames = np.arange(video_length)
+        spline_scores = spline(all_frames)
+
+        # Apply sigmoid function
+        def sigmoid(x):
+            return 1 / (1 + np.exp(-x))
+
+        adjusted_scores = np.maximum(1 / video_length, spline_scores)
+        p_distribution = sigmoid(adjusted_scores)
+
+        # Normalize the distribution
+        p_distribution /= p_distribution.sum()
+
+        return p_distribution
+
+    def update_frame_distribution(
+        self,
+        sampled_frame_indices: List[int],
+        confidence_maps: np.ndarray,
+        detected_objects_maps: List[List[List[str]]]
+    ) -> Tuple[List[float], List[List[str]]]:
+        """
+        Update the frame distribution based on detection results.
+
+        Args:
+            sampled_frame_indices (List[int]): Indices of sampled frames.
+            confidence_maps (np.ndarray): Confidence maps from detection.
+            detected_objects_maps (List[List[List[str]]]): Detected objects from detection.
+
+        Returns:
+            Tuple[List[float], List[List[str]]]: Frame confidences and detected objects.
+        """
+        confidence_map = confidence_maps[0]  # Only one image grid @TBD
+        detected_objects_map = detected_objects_maps[0]
+
+        grid_rows, grid_cols = self.image_grid_shape
+
+        frame_confidences = []
+        frame_detected_objects = []
+        for idx, frame_idx in enumerate(sampled_frame_indices):
+            # Calculate grid cell position
+            row = idx // grid_cols
+            col = idx % grid_cols
+            confidence = confidence_map[row, col]
+            detected_objects = detected_objects_map[idx]
+            frame_confidences.append(confidence)
+            frame_detected_objects.append(detected_objects)
+
+        # Update non-visiting frames and score distribution
+        for frame_idx, confidence in zip(sampled_frame_indices, frame_confidences):
+            self.non_visiting_frames[frame_idx] = 0  # Mark as visited
+            self.score_distribution[frame_idx] = confidence
+
+        # Update top 25% frames
+        self.update_top_25_with_window(frame_confidences, sampled_frame_indices)
+
+        # Update probability distribution
+        self.P = self.spline_keyframe_distribution(
+            self.non_visiting_frames,
+            self.score_distribution,
+            len(self.score_distribution)
+        )
+
+        # Store the updated distribution
+        self.store_score_distribution()
+
+        return frame_confidences, frame_detected_objects
+
+    ### --- Sampling Methods --- ###
+
+    def sample_frames(self, num_samples: int) -> Tuple[List[int], np.ndarray]:
+        """
+        Sample frames based on the current score distribution.
+
+        Args:
+            num_samples (int): Number of frames to sample.
+
+        Returns:
+            Tuple[List[int], np.ndarray]: Sampled frame indices and frame data.
+        """
+        if num_samples > self.total_frame_num:
+            num_samples = self.total_frame_num
+
+        # Adjust probabilities for non-visited frames
+        _P = (self.P + num_samples / self.total_frame_num) * self.non_visiting_frames
+        _P /= _P.sum()
+
+        # Sample frames
+        sampled_frame_secs = np.random.choice(
+            self.total_frame_num,
+            size=num_samples,
+            replace=False,
+            p=_P
+        )
+        sampled_frame_indices = [int(sec * self.raw_fps / self.fps) for sec in sampled_frame_secs]
+
+        # Read frames
+        frame_indices, frames = self.read_frame_batch(
+            video_path=self.video_path,
+            frame_indices=sampled_frame_indices
+        )
+
+        return sampled_frame_secs.tolist(), frames
+
+    ### --- Verification Methods --- ###
+
+    def verify_and_remove_target(
+        self,
+        frame_sec: int,
+        detected_objects: List[str],
+        confidence_threshold: float,
+    ) -> bool:
+        """
+        Verify target object detection in an individual frame and remove it from the target list if confirmed.
+
+        Args:
+            frame_sec (int): The timestamp of the frame in seconds.
+            detected_objects (List[str]): Objects detected in the grid image for this frame.
+            confidence_threshold (float): Threshold to confirm target detection.
+
+        Returns:
+            bool: True if a target was found and removed, False otherwise.
+        """
+        for target in list(self.remaining_targets):
+            if target in detected_objects:
+                frame_idx = int(frame_sec * self.raw_fps / self.fps)
+                # Read the individual frame
+                _, frame = self.read_frame_batch(self.video_path, [frame_idx])
+                frame = frame[0]  # Extract the frame from the list
+
+                # Perform detection on the individual frame
+                single_confidence_maps, single_detected_objects_maps = self.score_image_grids(
+                    [frame], (1, 1)
+                )
+                single_confidence = single_confidence_maps[0, 0, 0]
+                single_detected_objects = single_detected_objects_maps[0][0]
+                self.score_distribution[frame_sec] = single_confidence
+
+                # Check if target object confidence exceeds the threshold
+                if target in single_detected_objects and single_confidence > confidence_threshold:
+                    self.remaining_targets.remove(target)
+                    print(f"Found target '{target}' in frame {frame_idx}, score {single_confidence:.2f}")
+                    self.image_grid_iters.append([frame])
+                    self.detect_annotot_iters.append(self.owl.bbox_visualization(images=[frame], detections_inbatch=self.owl.detections_inbatch))
+                    self.detect_bbox_iters.append(self.owl.detections_inbatch)
+                    
+                    return True
+
+        return False
+
+    ### --- Visualization Methods --- ###
+
+    def plot_score_distribution(self, save_path: Optional[str] = None):
+        """
+        Plot the score distribution over time.
+
+        Args:
+            save_path (Optional[str]): File path to save the plot.
+        """
+        time_axis = np.linspace(0, self.duration, len(self.score_distribution))
+
+        plt.figure(figsize=(12, 6))
+        plt.plot(time_axis, self.score_distribution, label="Score Distribution")
+        plt.xlabel("Time (seconds)")
+        plt.ylabel("Score")
+        plt.title("Score Distribution Over Time")
+        plt.grid(True)
+        plt.legend()
+
+        if save_path:
+            plt.savefig(save_path, format='png', dpi=300)
+            print(f"Plot saved to {save_path}")
+
+        plt.show()
+
+    ### --- Main Search Logic --- ###
+
+    def search(self) -> Tuple[List[np.ndarray], List[float]]:
+        """
+        Perform the keyframe search based on object detection and dynamic sampling.
+
+        Returns:
+            Tuple[List[np.ndarray], List[float]]: Extracted keyframes and their timestamps.
+        """
+        K = self.search_nframes  # Number of keyframes to find
+            # Estimate the total number of iterations based on search_budget and frames per iteration
+        video_length = int(self.total_frame_num)
+        
+        # Initialize tqdm progress bar
+        progress_bar = tqdm(total=video_length, desc="Searching Iterations / video_length", unit="iter", dynamic_ncols=True)
+        
+        while self.remaining_targets and self.search_budget > 0:
+            grid_rows, grid_cols = self.image_grid_shape
+            num_frames_in_grid = grid_rows * grid_cols
+
+            # Sample frames based on the current distribution
+            sampled_frame_secs, frames = self.sample_frames(num_frames_in_grid)
+            self.search_budget -= num_frames_in_grid
+
+            # Create an image grid from the sampled frames
+            grid_image = self.create_image_grid(frames, grid_rows, grid_cols)
+
+            # Perform object detection on the image grid
+            confidence_maps, detected_objects_maps = self.score_image_grids(
+                images=[grid_image],
+                image_grids=self.image_grid_shape
+            )
+
+            # Update frame distributions based on detection results
+            frame_confidences, frame_detected_objects = self.update_frame_distribution(
+                sampled_frame_indices=sampled_frame_secs,
+                confidence_maps=confidence_maps,
+                detected_objects_maps=detected_objects_maps
+            )
+
+            # Verify and remove detected targets
+            for frame_sec, detected_objects in zip(sampled_frame_secs, frame_detected_objects):
+                self.verify_and_remove_target(
+                    frame_sec=frame_sec,
+                    detected_objects=detected_objects,
+                    confidence_threshold=self.confidence_threshold,
+                )
+            # Update the progress bar
+            progress_bar.update(1)
+        
+        # Close the progress bar once the loop is done
+        progress_bar.close()
+        # Select top K frames based on the score distribution
+        top_k_indices = np.argsort(self.score_distribution)[-K:][::-1]
+        top_k_frames = []
+        time_stamps = []
+
+        # Read and store the top K frames
+        for idx in top_k_indices:
+            frame_idx = int(idx * self.raw_fps / self.fps)
+            _, frame = self.read_frame_batch(self.video_path, [frame_idx])
+            top_k_frames.append(frame[0])
+            time_stamps.append(idx / self.fps)
+
+        return top_k_frames, time_stamps
+
+
+
+    def search_with_visualization(self) -> Tuple[List[np.ndarray], List[float]]:
+        """
+        Perform the keyframe search based on object detection and dynamic sampling.
+
+        Returns:
+            Tuple[List[np.ndarray], List[float]]: Extracted keyframes and their timestamps.
+        """
+
+
+        # Initialize history 
+        self.image_grid_iters = [] # iters, b, image
+        self.detect_annotot_iters = [] # iters, b, image
+        self.detect_bbox_iters = [] #iters, b, n_objects, xxyy
+            
+        K = self.search_nframes  # Number of keyframes to find
+            # Estimate the total number of iterations based on search_budget and frames per iteration
+        video_length = int(self.total_frame_num)
+        
+        # Initialize tqdm progress bar
+        progress_bar = tqdm(total=video_length, desc="Searching Iterations / video_length", unit="iter", dynamic_ncols=True)
+        
+        while self.remaining_targets and self.search_budget > 0:
+            grid_rows, grid_cols = self.image_grid_shape
+            num_frames_in_grid = grid_rows * grid_cols
+
+            # Sample frames based on the current distribution
+            sampled_frame_secs, frames = self.sample_frames(num_frames_in_grid)
+            self.search_budget -= num_frames_in_grid
+
+            # Create an image grid from the sampled frames
+            grid_image = self.create_image_grid(frames, grid_rows, grid_cols)
+            
+
+            # Perform object detection on the image grid
+            confidence_maps, detected_objects_maps = self.score_image_grids(
+                images=[grid_image],
+                image_grids=self.image_grid_shape
+            )
+
+            self.image_grid_iters.append([grid_image])
+            self.detect_annotot_iters.append(self.owl.bbox_visualization(images=[grid_image], detections_inbatch=self.owl.detections_inbatch))
+            self.detect_bbox_iters.append(self.owl.detections_inbatch)
+            
+            # Update frame distributions based on detection results
+            frame_confidences, frame_detected_objects = self.update_frame_distribution(
+                sampled_frame_indices=sampled_frame_secs,
+                confidence_maps=confidence_maps,
+                detected_objects_maps=detected_objects_maps
+            )
+
+            # Verify and remove detected targets
+            for frame_sec, detected_objects in zip(sampled_frame_secs, frame_detected_objects):
+                self.verify_and_remove_target(
+                    frame_sec=frame_sec,
+                    detected_objects=detected_objects,
+                    confidence_threshold=self.confidence_threshold,
+                )
+            # Update the progress bar
+            progress_bar.update(1)
+        
+        # Close the progress bar once the loop is done
+        progress_bar.close()
+        # Select top K frames based on the score distribution
+        top_k_indices = np.argsort(self.score_distribution)[-K:][::-1]
+        top_k_frames = []
+        time_stamps = []
+
+        # Read and store the top K frames
+        for idx in top_k_indices:
+            frame_idx = int(idx * self.raw_fps / self.fps)
+            _, frame = self.read_frame_batch(self.video_path, [frame_idx])
+            top_k_frames.append(frame[0])
+            time_stamps.append(idx / self.fps)
+
+        return top_k_frames, time_stamps
+
+
+
+
+
+# Example usage
+if __name__ == "__main__":
+    # Define video path and target objects
+    video_path = "/home/anabella/projects/MLLM/TSTAR/data/friend_clip_t.mp4"
+    query = "what is the color of the couch?"
+    target_objects = ["couch"]  # Target objects to find
+    cue_objects = ["table", "woman"]
+
+    # Create VideoSearcher instance
+    searcher = TStarSearcher(
+        video_path=video_path,
+        target_objects=target_objects,
+        cue_objects=cue_objects,
+        search_nframes=8,
+        image_grid_shape=(4, 4),
+        confidence_threshold=0.6
+    )
+
+    # Perform the search
+    all_frames, time_stamps = searcher.search()
+
+    # Process results
+    print(f"Found {len(all_frames)} frames, timestamps: {time_stamps}")
+
+    # Plot the score distribution
+    searcher.plot_score_distribution(save_path='./output/score/score_distribution.png')

TStar/interface_yolo.py

@@ -0,0 +1,279 @@
+
+import os
+import cv2
+import os.path as osp
+from mmengine.config import Config
+from mmengine.dataset import Compose
+from mmdet.apis import init_detector
+from mmdet.utils import get_test_pipeline_cfg
+# from mmengine.runner.amp import autocast
+from torch.amp import autocast
+import torch
+import supervision as sv
+from typing import Dict, Optional, Sequence, List
+
+import supervision as sv
+class LabelAnnotator(sv.LabelAnnotator):
+
+    @staticmethod
+    def resolve_text_background_xyxy(
+        center_coordinates,
+        text_wh,
+        position,
+    ):
+        center_x, center_y = center_coordinates
+        text_w, text_h = text_wh
+        return center_x, center_y, center_x + text_w, center_y + text_h
+
+
+class YoloInterface:
+    def __init__(self):
+        """
+        Initialize the YOLO-World model with the given configuration and checkpoint.
+
+        Args:
+        """
+        
+     
+        pass
+    def set_BBoxAnnotator(self):
+        self.BOUNDING_BOX_ANNOTATOR = sv.BoundingBoxAnnotator(thickness=1)
+        # MASK_ANNOTATOR = sv.MaskAnnotator()
+        self.LABEL_ANNOTATOR = LabelAnnotator(text_padding=4,
+                                        text_scale=0.5,
+                                        text_thickness=1)
+
+class YoloWorldInterface(YoloInterface):
+    def __init__(self, config_path: str, checkpoint_path: str, device: str = "cuda:0"):
+        """
+        Initialize the YOLO-World model with the given configuration and checkpoint.
+
+        Args:
+            config_path (str): Path to the model configuration file.
+            checkpoint_path (str): Path to the model checkpoint.
+            device (str): Device to run the model on (e.g., 'cuda:0', 'cpu').
+        """
+        self.config_path = config_path
+        self.checkpoint_path = checkpoint_path
+        self.device = device
+
+        # Load configuration
+        cfg = Config.fromfile(config_path)
+        cfg.work_dir = osp.join('./work_dirs', osp.splitext(osp.basename(config_path))[0])
+        cfg.load_from = checkpoint_path
+
+        # Initialize the model
+        self.model = init_detector(cfg, checkpoint=checkpoint_path, device=device)
+        self.set_BBoxAnnotator()
+
+        # Initialize the test pipeline
+        # build test pipeline
+        self.model.cfg.test_dataloader.dataset.pipeline[
+            0].type = 'mmdet.LoadImageFromNDArray'
+        self.test_pipeline = Compose(self.model.cfg.test_dataloader.dataset.pipeline)
+
+        
+
+    def reparameterize_object_list(self, target_objects: List[str], cue_objects: List[str]):
+        """
+        Reparameterize the detect object list to be used by the YOLO model.
+
+        Args:
+            target_objects (List[str]): List of target object names.
+            cue_objects (List[str]): List of cue object names.
+        """
+        # Combine target objects and cue objects into the final text format
+        combined_texts = target_objects + cue_objects
+
+        # Format the text prompts for the YOLO model
+        self.texts = [[obj.strip()] for obj in combined_texts] + [[' ']]
+
+        # Reparameterize the YOLO model with the provided text prompts
+        self.model.reparameterize(self.texts)
+
+
+    def inference(self, image: str, max_dets: int = 100, score_threshold: float = 0.3, use_amp: bool = False):
+        """
+        Run inference on a single image.
+
+        Args:
+            image (str): Path to the image.
+            max_dets (int): Maximum number of detections to keep.
+            score_threshold (float): Score threshold for filtering detections.
+            use_amp (bool): Whether to use mixed precision for inference.
+
+        Returns:
+            sv.Detections: Detection results.
+        """
+        # Prepare data for inference
+        data_info = dict(img_id=0, img_path=image, texts=self.texts)
+        data_info = self.test_pipeline(data_info)
+        data_batch = dict(inputs=data_info['inputs'].unsqueeze(0),
+                          data_samples=[data_info['data_samples']])
+
+        # Run inference
+        with autocast(enabled=use_amp), torch.no_grad():
+            output = self.model.test_step(data_batch)[0]
+            pred_instances = output.pred_instances
+            pred_instances = pred_instances[pred_instances.scores.float() > score_threshold]
+
+        if len(pred_instances.scores) > max_dets:
+            indices = pred_instances.scores.float().topk(max_dets)[1]
+            pred_instances = pred_instances[indices]
+
+        pred_instances = pred_instances.cpu().numpy()
+
+        # Process detections
+        detections = sv.Detections(
+            xyxy=pred_instances['bboxes'],
+            class_id=pred_instances['labels'],
+            confidence=pred_instances['scores'],
+            mask=pred_instances.get('masks', None)
+        )
+        return detections
+    
+    def inference_detector(self, images, max_dets=50, score_threshold=0.2, use_amp: bool = False):
+        data_info = dict(img_id=0, img=images[0], texts=self.texts) #TBD for batch searching
+        data_info = self.test_pipeline(data_info)
+        data_batch = dict(inputs=data_info['inputs'].unsqueeze(0),
+                        data_samples=[data_info['data_samples']])
+        detections_inbatch = []
+        with torch.no_grad():
+            outputs = self.model.test_step(data_batch)
+            # cover to searcher interface format
+            
+            for output in outputs:
+                pred_instances = output.pred_instances
+                pred_instances = pred_instances[pred_instances.scores.float() >
+                                                score_threshold]
+                if len(pred_instances.scores) > max_dets:
+                    indices = pred_instances.scores.float().topk(max_dets)[1]
+                    pred_instances = pred_instances[indices]
+
+                output.pred_instances = pred_instances
+
+                if 'masks' in pred_instances:
+                    masks = pred_instances['masks']
+                else:
+                    masks = None
+                pred_instances = pred_instances.cpu().numpy()
+                detections = sv.Detections(xyxy=pred_instances['bboxes'],
+                    class_id=pred_instances['labels'],
+                    confidence=pred_instances['scores'],
+                    mask=masks)
+                detections_inbatch.append(detections)
+        self.detect_outputs_raw = outputs
+        self.detections_inbatch = detections_inbatch
+        return detections_inbatch
+
+    def bbox_visualization(self, images, detections_inbatch):
+        anno_images = []
+        # detections_inbatch = self.detections_inbatch
+        for b, detections in enumerate(detections_inbatch):
+            texts = self.texts
+            labels = [
+                f"{texts[class_id][0]} {confidence:0.2f}" for class_id, confidence in
+                zip(detections.class_id, detections.confidence)
+            ]
+
+        
+            index = len(detections_inbatch) -1 
+            image = images[index]
+            anno_image = image.copy()
+  
+    
+            anno_image = self.BOUNDING_BOX_ANNOTATOR.annotate(anno_image, detections)
+            anno_image = self.LABEL_ANNOTATOR.annotate(anno_image, detections, labels=labels)
+            anno_images.append(anno_image)
+        
+        return anno_images
+
+
+
+import torch
+from typing import List
+import supervision as sv  # 确保已安装 Supervision 库
+import os.path as osp
+
+class YoloV5Interface(YoloInterface):
+    def __init__(self,config_path="ultralytics/yolov5", checkpoint_path: str = 'yolov5s', device: str = 'cuda:0'):
+        """
+        初始化 YOLOv5 模型。
+
+        Args:
+            model_name (str): YOLOv5 模型变体名称（如 'yolov5s', 'yolov5m', 'yolov5l', 'yolov5x'）。
+            device (str): 运行模型的设备（如 'cuda:0', 'cpu'）。
+        """
+        self.device = device
+        self.model = torch.hub.load("ultralytics/yolov5", "yolov5s", pretrained=True)
+        
+        self.model.to(self.device)
+        self.model.eval()
+        self.target_classes = None  # 用于存储目标类别列表
+
+        self.texts = None
+        self.test_pipeline = None
+
+    def reparameterize_object_list(self, target_objects: List[str], cue_objects: List[str]):
+        """
+        重新参数化检测对象列表，以便在推理时使用。
+
+        Args:
+            target_objects (List[str]): 目标对象名称列表。
+            cue_objects (List[str]): 线索对象名称列表。
+        """
+        # 合并目标对象和线索对象
+        combined_objects = target_objects + cue_objects
+        self.target_classes = combined_objects
+
+    def inference(self, images: str, max_dets: int = 100, score_threshold: float = 0.3, use_amp: bool = False):
+        """
+        对单张图像运行推理。
+
+        Args:
+            image (str): 图像路径。
+            max_dets (int): 保留的最大检测数量。
+            score_threshold (float): 过滤检测的分数阈值。
+            use_amp (bool): 是否使用混合精度进行推理。
+
+        Returns:
+            sv.Detections: 检测结果。
+        """
+        results = self.model(images, size=640)  # 可以根据需要调整输入尺寸
+
+        # 提取检测结果（假设批量大小为 1）
+        detections_batch = results.pred  # B tensors of shape (N, 6) [x1, y1, x2, y2, confidence, class]
+
+        # 应用分数阈值
+        # 用于存储每个批次过滤后的检测结果
+        filtered_detections = []
+
+        for detections in detections_batch:
+            # 应用分数阈值，过滤掉 confidence <= score_threshold 的检测
+            detections = detections[detections[:, 4] > score_threshold]
+            # 如果设置了 topk，截取前 topk 个检测
+            if len(detections) > max_dets:
+                detections = detections[:max_dets]
+            # 如果设置了目标类别，过滤检测结果
+            if self.target_classes is not None:
+                # 获取所有类别名称
+                class_names = self.model.names
+                # 获取目标类别的类别ID
+                target_class_ids = [i for i, name in class_names.items() if name in self.target_classes]
+
+                
+                # 过滤检测结果
+                detections = detections[[cls in target_class_ids for cls in detections[:, 5]]]
+                # 转换为 Supervision 库的 Detections 对象
+                detections = sv.Detections(
+                    xyxy=detections[:, :4].cpu().numpy(),
+                    confidence=detections[:, 4].cpu().numpy(),
+                    class_id=detections[:, 5].cpu().numpy().astype(int)
+                )
+                
+
+                filtered_detections.append(detections)
+
+        return filtered_detections
+
+

TStar/utils.py

@@ -0,0 +1,105 @@
+
+
+try:
+    from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+    from llava.constants import IGNORE_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IMAGE_TOKEN_INDEX
+except ImportError:
+    print("Warning: LLAVA constants not found, using non-values.")
+from typing import List
+import math
+from typing import List, Dict
+from PIL import Image
+import base64
+import io
+import numpy as np
+from contextlib import contextmanager
+
+try:
+    import cv2
+except ImportError:
+    cv2 = None
+    print("Warning: OpenCV is not installed, video frame extraction will not work.")
+
+
+
+def encode_image_to_base64(image) -> str:
+    """
+    Convert an image (PIL.Image or numpy.ndarray) to a Base64 encoded string.
+    """
+    try:
+        # If the input is a numpy array, convert it to a PIL Image
+        if isinstance(image, np.ndarray):
+            image = Image.fromarray(image)
+
+        # Ensure it's a PIL Image before proceeding
+        if not isinstance(image, Image.Image):
+            raise ValueError("Input must be a PIL.Image or numpy.ndarray")
+
+        # Encode the image to Base64
+        buffered = io.BytesIO()
+        image.save(buffered, format="JPEG")
+        return base64.b64encode(buffered.getvalue()).decode("utf-8")
+    except Exception as e:
+        raise ValueError(f"Error encoding image: {str(e)}")
+
+def load_video_frames(video_path: str, num_frames: int = 8) -> List[Image.Image]:
+    """
+    从视频中读取 num_frames 帧并返回 PIL.Image 列表。
+    """
+    if cv2 is None:
+        raise ImportError("OpenCV is not installed, cannot load video frames.")
+
+    frames = []
+    cap = cv2.VideoCapture(video_path)
+    if not cap.isOpened():
+        raise ValueError(f"Cannot open video: {video_path}")
+
+    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    if total_frames == 0:
+        cap.release()
+        raise ValueError("Video has zero frames or could not retrieve frame count.")
+    
+    num_frames = min(num_frames, total_frames)
+    step = total_frames / num_frames
+
+    for i in range(num_frames):
+        frame_index = int(math.floor(i * step))
+        cap.set(cv2.CAP_PROP_POS_FRAMES, frame_index)
+        ret, frame = cap.read()
+        if not ret:
+            break
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        frames.append(Image.fromarray(frame_rgb))
+
+    cap.release()
+    return frames
+
+
+def save_as_gif(images, output_gif_path):
+    from PIL import Image
+    import os
+
+    fps = 1  # 设置帧率为 1
+    duration = int(1000 / fps)  # GIF 每帧显示时间，单位为毫秒
+
+    # 将每一帧图像转换为 PIL 图像
+    pil_images = [Image.fromarray(img.astype('uint8')) for img in images]
+    
+    # 保存为 GIF
+    pil_images[0].save(
+        output_gif_path, 
+        save_all=True, 
+        append_images=pil_images[1:], 
+        duration=duration, 
+        loop=0  # 设置循环播放（0 为无限循环）
+    )
+    print(f"Saved GIF: {output_gif_path}")
+
+@contextmanager
+def NoWarning():
+    from logging import logger
+    logger.set_level(logger.ERROR)
+    try:
+        yield
+    finally:
+        logger.set_level(logger.INFO)

YOLO-World/.dockerignore

@@ -0,0 +1,2 @@
+docs
+Dockerfile

YOLO-World/.gitattributes

@@ -0,0 +1,34 @@
+# Basic .gitattributes for a python repo.
+
+# Source files
+# ============
+*.pxd    text diff=python
+*.py     text diff=python
+*.py3    text diff=python
+*.pyw    text diff=python
+*.pyx    text diff=python
+*.pyz    text diff=python
+*.pyi    text diff=python
+
+# Binary files
+# ============
+*.db     binary
+*.p      binary
+*.pkl    binary
+*.pickle binary
+*.pyc    binary export-ignore
+*.pyo    binary export-ignore
+*.pyd    binary
+
+# Jupyter notebook
+*.ipynb  text eol=lf
+
+# Others
+*           text=auto
+*.txt		text
+*.sh		text eol=lf
+
+# Note: .db, .p, and .pkl files are associated
+# with the python modules ``pickle``, ``dbm.*``,
+# ``shelve``, ``marshal``, ``anydbm``, & ``bsddb``
+# (among others).

YOLO-World/.gitignore

@@ -0,0 +1,127 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/en/_build/
+docs/zh_cn/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+# data/
+# data
+.vscode
+.idea
+.DS_Store
+
+# custom
+*.pkl
+*.pkl.json
+*.log.json
+docs/modelzoo_statistics.md
+mmdet/.mim
+work_dirs
+
+# Pytorch
+*.pth
+*.py~
+*.sh~
+
+# venus
+venus_run.sh
+

YOLO-World/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/mmyolo"]
+	path = third_party/mmyolo
+	url = https://github.com/onuralpszr/mmyolo.git

YOLO-World/Dockerfile

@@ -0,0 +1,43 @@
+FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
+
+ARG MODEL="yolo_world_l_dual_vlpan_l2norm_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py"
+ARG WEIGHT="yolo_world_l_clip_base_dual_vlpan_2e-3adamw_32xb16_100e_o365_goldg_train_pretrained-0e566235.pth"
+
+ENV FORCE_CUDA="1"
+ENV MMCV_WITH_OPS=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3-pip     \
+    libgl1-mesa-glx \
+    libsm6          \
+    libxext6        \
+    libxrender-dev  \
+    libglib2.0-0    \
+    git             \
+    python3-dev     \
+    python3-wheel
+
+RUN pip3 install --upgrade pip \
+    && pip3 install   \
+        gradio        \
+        opencv-python \
+        supervision   \
+        mmengine      \
+        setuptools    \
+        openmim       \
+    && mim install mmcv==2.0.0 \
+    && pip3 install --no-cache-dir --index-url https://download.pytorch.org/whl/cu118 \
+        wheel         \
+        torch         \
+        torchvision   \
+        torchaudio
+
+COPY . /yolo
+WORKDIR /yolo
+
+RUN pip3 install -e .
+
+RUN curl -o weights/$WEIGHT -L https://huggingface.co/wondervictor/YOLO-World/resolve/main/$WEIGHT
+
+ENTRYPOINT [ "python3", "demo.py" ]
+CMD ["configs/pretrain/$MODEL", "weights/$WEIGHT"]

YOLO-World/LICENSE

@@ -0,0 +1,1347 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

YOLO-World/README.md

@@ -0,0 +1,238 @@
+<div align="center">
+<img src="./assets/yolo_logo.png" width=60%>
+<br>
+<a href="https://scholar.google.com/citations?hl=zh-CN&user=PH8rJHYAAAAJ">Tianheng Cheng</a><sup><span>2,3,*</span></sup>, 
+<a href="https://linsong.info/">Lin Song</a><sup><span>1,📧,*</span></sup>,
+<a href="https://yxgeee.github.io/">Yixiao Ge</a><sup><span>1,🌟,2</span></sup>,
+<a href="http://eic.hust.edu.cn/professor/liuwenyu/"> Wenyu Liu</a><sup><span>3</span></sup>,
+<a href="https://xwcv.github.io/">Xinggang Wang</a><sup><span>3,📧</span></sup>,
+<a href="https://scholar.google.com/citations?user=4oXBp9UAAAAJ&hl=en">Ying Shan</a><sup><span>1,2</span></sup>
+</br>
+
+\* Equal contribution 🌟 Project lead 📧 Corresponding author
+
+<sup>1</sup> Tencent AI Lab,  <sup>2</sup> ARC Lab, Tencent PCG
+<sup>3</sup> Huazhong University of Science and Technology
+<br>
+<div>
+
+[![arxiv paper](https://img.shields.io/badge/Project-Page-green)](https://wondervictor.github.io/)
+[![arxiv paper](https://img.shields.io/badge/arXiv-Paper-red)](https://arxiv.org/abs/2401.17270)
+<a href="https://colab.research.google.com/github/AILab-CVC/YOLO-World/blob/master/inference.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a>
+[![demo](https://img.shields.io/badge/🤗HugginngFace-Spaces-orange)](https://huggingface.co/spaces/stevengrove/YOLO-World)
+[![Replicate](https://replicate.com/zsxkib/yolo-world/badge)](https://replicate.com/zsxkib/yolo-world)
+[![hfpaper](https://img.shields.io/badge/🤗HugginngFace-Paper-yellow)](https://huggingface.co/papers/2401.17270)
+[![license](https://img.shields.io/badge/License-GPLv3.0-blue)](LICENSE)
+[![yoloworldseg](https://img.shields.io/badge/YOLOWorldxEfficientSAM-🤗Spaces-orange)](https://huggingface.co/spaces/SkalskiP/YOLO-World)
+[![yologuide](https://img.shields.io/badge/📖Notebook-roboflow-purple)](https://supervision.roboflow.com/develop/notebooks/zero-shot-object-detection-with-yolo-world)
+[![deploy](https://media.roboflow.com/deploy.svg)](https://inference.roboflow.com/foundation/yolo_world/)
+
+</div>
+</div>
+
+## Notice
+
+**YOLO-World is still under active development!**
+
+We recommend that everyone **use English to communicate on issues**, as this helps developers from around the world discuss, share experiences, and answer questions together.
+
+For business licensing and other related inquiries, don't hesitate to contact `yixiaoge@tencent.com`.
+
+## 🔥 Updates 
+`[2024-11-5]`: We update the `YOLO-World-Image` and you can try it at HuggingFace [YOLO-World-Image (Preview Version)](https://huggingface.co/spaces/wondervictor/YOLO-World-Image). It's a *preview* version and we are still improving it! Detailed documents about training and few-shot inference are coming soon.\
+`[2024-7-8]`: YOLO-World now has been integrated into [ComfyUI](https://github.com/StevenGrove/ComfyUI-YOLOWorld)! Come and try adding YOLO-World to your workflow now! You can access it at [StevenGrove/ComfyUI-YOLOWorld](https://github.com/StevenGrove/ComfyUI-YOLOWorld)!  
+`[2024-5-18]:` YOLO-World models have been [integrated with the FiftyOne computer vision toolkit](https://docs.voxel51.com/integrations/ultralytics.html#open-vocabulary-detection) for streamlined open-vocabulary inference across image and video datasets.  
+`[2024-5-16]:` Hey guys! Long time no see! This update contains (1) [fine-tuning guide](https://github.com/AILab-CVC/YOLO-World?#highlights--introduction) and (2) [TFLite Export](./docs/tflite_deploy.md) with INT8 Quantization.  
+`[2024-5-9]:` This update contains the real [`reparameterization`](./docs/reparameterize.md) 🪄, and it's better for fine-tuning on custom datasets and improves the training/inference efficiency 🚀!  
+`[2024-4-28]:` Long time no see! This update contains bugfixs and improvements: (1) ONNX demo; (2) image demo (support tensor input); (2) new pre-trained models; (3) image prompts; (4) simple version for fine-tuning / deployment; (5) guide for installation (include a `requirements.txt`).  
+`[2024-3-28]:` We provide: (1) more high-resolution pre-trained models (e.g., S, M, X) ([#142](https://github.com/AILab-CVC/YOLO-World/issues/142)); (2) pre-trained models with CLIP-Large text encoders. Most importantly, we preliminarily fix the **fine-tuning without `mask-refine`** and explore a new fine-tuning setting ([#160](https://github.com/AILab-CVC/YOLO-World/issues/160),[#76](https://github.com/AILab-CVC/YOLO-World/issues/76)). In addition, fine-tuning YOLO-World with `mask-refine` also obtains significant improvements, check more details in [configs/finetune_coco](./configs/finetune_coco/).  
+`[2024-3-16]:` We fix the bugs about the demo ([#110](https://github.com/AILab-CVC/YOLO-World/issues/110),[#94](https://github.com/AILab-CVC/YOLO-World/issues/94),[#129](https://github.com/AILab-CVC/YOLO-World/issues/129), [#125](https://github.com/AILab-CVC/YOLO-World/issues/125)) with visualizations of segmentation masks, and release [**YOLO-World with Embeddings**](./docs/prompt_yolo_world.md), which supports prompt tuning, text prompts and image prompts.  
+`[2024-3-3]:` We add the **high-resolution YOLO-World**, which supports `1280x1280` resolution with higher accuracy and better performance for small objects!  
+`[2024-2-29]:` We release the newest version of [ **YOLO-World-v2**](./docs/updates.md) with higher accuracy and faster speed! We hope the community can join us to improve YOLO-World!  
+`[2024-2-28]:` Excited to announce that YOLO-World has been accepted by **CVPR 2024**! We're continuing to make YOLO-World faster and stronger, as well as making it better to use for all.  
+`[2024-2-22]:` We sincerely thank [RoboFlow](https://roboflow.com/) and [@Skalskip92](https://twitter.com/skalskip92) for the [**Video Guide**](https://www.youtube.com/watch?v=X7gKBGVz4vs) about YOLO-World, nice work!  
+`[2024-2-18]:` We thank [@Skalskip92](https://twitter.com/skalskip92) for developing the wonderful segmentation demo via connecting YOLO-World and EfficientSAM. You can try it now at the [🤗 HuggingFace Spaces](https://huggingface.co/spaces/SkalskiP/YOLO-World).   
+`[2024-2-17]:` The largest model **X** of YOLO-World is released, which achieves better zero-shot performance!   
+`[2024-2-17]:` We release the code & models for **YOLO-World-Seg** now! YOLO-World now supports open-vocabulary / zero-shot object segmentation!  
+`[2024-2-15]:` The pre-traind YOLO-World-L with CC3M-Lite is released!     
+`[2024-2-14]:` We provide the [`image_demo`](demo.py) for inference on images or directories.   
+`[2024-2-10]:` We provide the [fine-tuning](./docs/finetuning.md) and [data](./docs/data.md) details for fine-tuning YOLO-World on the COCO dataset or the custom datasets!  
+`[2024-2-3]:` We support the `Gradio` demo now in the repo and you can build the YOLO-World demo on your own device!  
+`[2024-2-1]:` We've released the code and weights of YOLO-World now!  
+`[2024-2-1]:` We deploy the YOLO-World demo on [HuggingFace 🤗](https://huggingface.co/spaces/stevengrove/YOLO-World), you can try it now!  
+`[2024-1-31]:` We are excited to launch **YOLO-World**, a cutting-edge real-time open-vocabulary object detector.  
+
+
+## TODO
+
+YOLO-World is under active development and please stay tuned ☕️! 
+If you have suggestions📃 or ideas💡,**we would love for you to bring them up in the [Roadmap](https://github.com/AILab-CVC/YOLO-World/issues/109)** ❤️!
+> YOLO-World 目前正在积极开发中📃，如果你有建议或者想法💡，**我们非常希望您在 [Roadmap](https://github.com/AILab-CVC/YOLO-World/issues/109) 中提出来** ❤️！
+
+## [FAQ (Frequently Asked Questions)](https://github.com/AILab-CVC/YOLO-World/discussions/149)
+
+We have set up an FAQ about YOLO-World in the discussion on GitHub. We hope everyone can raise issues or solutions during use here, and we also hope that everyone can quickly find solutions from it.
+
+> 我们在GitHub的discussion中建立了关于YOLO-World的常见问答，这里将收集一些常见问题，同时大家可以在此提出使用中的问题或者解决方案，也希望大家能够从中快速寻找到解决方案
+
+
+## Highlights & Introduction
+
+This repo contains the PyTorch implementation, pre-trained weights, and pre-training/fine-tuning code for YOLO-World.
+
+* YOLO-World is pre-trained on large-scale datasets, including detection, grounding, and image-text datasets.
+
+* YOLO-World is the next-generation YOLO detector, with a strong open-vocabulary detection capability and grounding ability.
+
+* YOLO-World presents a *prompt-then-detect* paradigm for efficient user-vocabulary inference, which re-parameterizes vocabulary embeddings as parameters into the model and achieve superior inference speed. You can try to export your own detection model without extra training or fine-tuning in our [online demo](https://huggingface.co/spaces/stevengrove/YOLO-World)!
+
+
+<div align="center">
+<img width=800px src="./assets/yolo_arch.png">
+</div>
+## Model Zoo
+
+We've pre-trained YOLO-World-S/M/L from scratch and evaluate on the `LVIS val-1.0` and `LVIS minival`. We provide the pre-trained model weights and training logs for applications/research or re-producing the results.
+
+### Zero-shot Inference on LVIS dataset
+
+<div><font size=2>
+
+| model                                                                                                                | Pre-train Data       | Size | AP<sup>mini</su> | AP<sub>r</sub> | AP<sub>c</sub> | AP<sub>f</sub> | AP<sup>val</su> | AP<sub>r</sub> | AP<sub>c</sub> | AP<sub>f</sub> |                                                                                        weights                                                                                         |
+| :------------------------------------------------------------------------------------------------------------------- | :------------------- | :----------------- | :--------------: | :------------: | :------------: | :------------: | :-------------: | :------------: | :------------: | :------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
+| [YOLO-Worldv2-S](./configs/pretrain/yolo_world_v2_s_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py) | O365+GoldG | 640 | 22.7 | 16.3 | 20.8 | 25.5 |  17.3 | 11.3 | 14.9 | 22.7 |[HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_s_obj365v1_goldg_pretrain-55b943ea.pth)|
+| [YOLO-Worldv2-S](./configs/pretrain/yolo_world_v2_s_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_1280ft_lvis_minival.py) | O365+GoldG | 1280&#x1F538; | 24.1 | 18.7 | 22.0 | 26.9 |  18.8 | 14.1 | 16.3 | 23.8 |[HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_s_obj365v1_goldg_pretrain_1280ft-fc4ff4f7.pth)|  
+| [YOLO-Worldv2-M](./configs/pretrain/yolo_world_v2_m_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py)  | O365+GoldG | 640 | 30.0 | 25.0  | 27.2 | 33.4 | 23.5 | 17.1 | 20.0 | 30.1 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_m_obj365v1_goldg_pretrain-c6237d5b.pth)| 
+| [YOLO-Worldv2-M](./configs/pretrain/yolo_world_v2_m_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_1280ft_lvis_minival.py) | O365+GoldG | 1280&#x1F538; | 31.6 | 24.5  | 29.0 | 35.1 | 25.3 | 19.3 | 22.0 | 31.7 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_m_obj365v1_goldg_pretrain_1280ft-77d0346d.pth)| 
+| [YOLO-Worldv2-L](./configs/pretrain/yolo_world_v2_l_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py) | O365+GoldG | 640 | 33.0 | 22.6 | 32.0 | 35.8 | 26.0 | 18.6 | 23.0 | 32.6 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_l_obj365v1_goldg_pretrain-a82b1fe3.pth)| 
+| 🔥 [YOLO-Worldv2-L](./configs/pretrain/yolo_world_v2_l_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py) | O365+GoldG+CCLiteV2 | 640 | 33.4 | 23.1 | 31.9 | 36.6 | 26.6 | 20.3 | 23.2 | 33.2 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/resolve/main/yolo_world_v2_l_obj365v1_goldg_cc3mv2_pretrain-2f3a4a22.pth)| 
+| [YOLO-Worldv2-L](./configs/pretrain/yolo_world_v2_l_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_1280ft_lvis_minival.py) | O365+GoldG | 1280&#x1F538; | 34.6 | 29.2 | 32.8 | 37.2 | 27.6 | 21.9 | 24.2 | 34.0 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_l_obj365v1_goldg_pretrain_1280ft-9babe3f6.pth)| 
+| [YOLO-Worldv2-L (CLIP-Large)](./configs/pretrain/yolo_world_v2_l_clip_large_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py) 🔥  | O365+GoldG | 640 | 34.0 | 22.0 | 32.6 | 37.4 | 27.1 | 19.9 | 23.9 | 33.9 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_l_clip_large_o365v1_goldg_pretrain-8ff2e744.pth)|
+| [YOLO-Worldv2-L (CLIP-Large)](./configs/pretrain/yolo_world_v2_l_clip_large_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_800ft_lvis_minival.py) 🔥  | O365+GoldG | 800&#x1F538; | 35.5 | 28.3 | 33.2 | 38.8 | 28.6 | 22.0 | 25.1 | 35.4 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_l_clip_large_o365v1_goldg_pretrain_800ft-9df82e55.pth)|
+| [YOLO-Worldv2-L](./configs/pretrain/yolo_world_v2_l_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py) | O365+GoldG+CC3M-Lite | 640 | 32.9 | 25.3 | 31.1 | 35.8 | 26.1 | 20.6 | 22.6 | 32.3 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_l_obj365v1_goldg_cc3mlite_pretrain-ca93cd1f.pth)|
+| [YOLO-Worldv2-X](./configs/pretrain/yolo_world_v2_x_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py) | O365+GoldG+CC3M-Lite | 640 | 35.4 | 28.7 | 32.9 | 38.7 | 28.4 | 20.6 | 25.6 | 35.0 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_x_obj365v1_goldg_cc3mlite_pretrain-8698fbfa.pth) |
+| 🔥 [YOLO-Worldv2-X]() |  O365+GoldG+CC3M-Lite | 1280&#x1F538; | 37.4 | 30.5 | 35.2 | 40.7  | 29.8 | 21.1 | 26.8 | 37.0 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_x_obj365v1_goldg_cc3mlite_pretrain_1280ft-14996a36.pth) |
+| [YOLO-Worldv2-XL](./configs/pretrain/yolo_world_v2_xl_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py) | O365+GoldG+CC3M-Lite | 640 | 36.0 | 25.8 | 34.1 | 39.5 | 29.1 | 21.1 | 26.3 | 35.8 | [HF Checkpoints 🤗](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_xl_obj365v1_goldg_cc3mlite_pretrain-5daf1395.pth) |
+
+</font>
+</div>
+
+**NOTE:**
+1. AP<sup>mini</sup>: evaluated on LVIS `minival`.
+3. AP<sup>val</sup>: evaluated on LVIS `val 1.0`.
+4. [HuggingFace Mirror](https://hf-mirror.com/) provides the mirror of HuggingFace, which is a choice for users who are unable to reach.
+5. &#x1F538;: fine-tuning models with the pre-trained data.
+
+**Pre-training Logs:**
+
+We provide the pre-training logs of `YOLO-World-v2`. Due to the unexpected errors of the local machines, the training might be interrupted several times.
+
+| Model | YOLO-World-v2-S | YOLO-World-v2-M  | YOLO-World-v2-L | YOLO-World-v2-X |
+| :---  | :-------------: | :--------------: | :-------------: | :-------------: |
+|Pre-training Log | [Part-1](https://drive.google.com/file/d/1oib7pKfA2h1U_5-85H_s0Nz8jWd0R-WP/view?usp=drive_link), [Part-2](https://drive.google.com/file/d/11cZ6OZy80VTvBlZy3kzLAHCxx5Iix5-n/view?usp=drive_link) | [Part-1](https://drive.google.com/file/d/1E6vYSS8kBipGc8oQnsjAfeUAx8I9yOX7/view?usp=drive_link), [Part-2](https://drive.google.com/file/d/1fbM7vt2tgSeB8o_7tUDofWvpPNSViNj5/view?usp=drive_link) | [Part-1](https://drive.google.com/file/d/1Tola1QGJZTL6nGy3SBxKuknfNfREDm8J/view?usp=drive_link), [Part-2](https://drive.google.com/file/d/1mTBXniioUb0CdctCG4ckIU6idGo0NnH8/view?usp=drive_link) |  [Final part](https://drive.google.com/file/d/1aEUA_EPQbXOrpxHTQYB6ieGXudb1PLpd/view?usp=drive_link)| 
+
+
+## Getting started
+
+### 1. Installation
+
+YOLO-World is developed based on `torch==1.11.0` `mmyolo==0.6.0` and `mmdetection==3.0.0`. Check more details about `requirements` and `mmcv` in [docs/installation](./docs/installation.md).
+
+#### Clone Project 
+
+```bash
+git clone --recursive https://github.com/AILab-CVC/YOLO-World.git
+```
+#### Install
+
+```bash
+pip install torch wheel -q
+pip install -e .
+```
+
+### 2. Preparing Data
+
+We provide the details about the pre-training data in [docs/data](./docs/data.md).
+
+
+## Training & Evaluation
+
+We adopt the default [training](./tools/train.py) or [evaluation](./tools/test.py) scripts of [mmyolo](https://github.com/open-mmlab/mmyolo).
+We provide the configs for pre-training and fine-tuning in `configs/pretrain` and `configs/finetune_coco`.
+Training YOLO-World is easy:
+
+```bash
+chmod +x tools/dist_train.sh
+# sample command for pre-training, use AMP for mixed-precision training
+./tools/dist_train.sh configs/pretrain/yolo_world_l_t2i_bn_2e-4_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py 8 --amp
+```
+**NOTE:** YOLO-World is pre-trained on 4 nodes with 8 GPUs per node (32 GPUs in total). For pre-training, the `node_rank` and `nnodes` for multi-node training should be specified. 
+
+Evaluating YOLO-World is also easy:
+
+```bash
+chmod +x tools/dist_test.sh
+./tools/dist_test.sh path/to/config path/to/weights 8
+```
+
+**NOTE:** We mainly evaluate the performance on LVIS-minival for pre-training.
+
+## Fine-tuning YOLO-World
+
+<div align="center">
+<img src="./assets/finetune_yoloworld.png" width=800px>
+</div>
+
+
+<div align="center">
+<b><p>Chose your pre-trained YOLO-World and Fine-tune it!</p></b> 
+</div>
+
+
+YOLO-World supports **zero-shot inference**, and three types of **fine-tuning recipes**: **(1) normal fine-tuning**, **(2) prompt tuning**, and **(3) reparameterized fine-tuning**.
+
+* Normal Fine-tuning: we provide the details about fine-tuning YOLO-World in [docs/fine-tuning](./docs/finetuning.md).
+
+* Prompt Tuning: we provide more details ahout prompt tuning in [docs/prompt_yolo_world](./docs/prompt_yolo_world.md).
+
+* Reparameterized Fine-tuning: the reparameterized YOLO-World is more suitable for specific domains far from generic scenes. You can find more details in [docs/reparameterize](./docs/reparameterize.md).
+
+## Deployment
+
+We provide the details about deployment for downstream applications in [docs/deployment](./docs/deploy.md).
+You can directly download the ONNX model through the online [demo](https://huggingface.co/spaces/stevengrove/YOLO-World) in Huggingface Spaces 🤗.
+
+- [x] ONNX export and demo: [docs/deploy](https://github.com/AILab-CVC/YOLO-World/blob/master/docs/deploy.md)
+- [x] TFLite and INT8 Quantization: [docs/tflite_deploy](https://github.com/AILab-CVC/YOLO-World/blob/master/docs/tflite_deploy.md)
+- [ ] TensorRT: coming soon.
+- [ ] C++: coming soon.
+
+## Demo
+
+See [`demo`](./demo) for more details
+
+- [x] `gradio_demo.py`: Gradio demo, ONNX export
+- [x] `image_demo.py`: inference with images or a directory of images
+- [x] `simple_demo.py`: a simple demo of YOLO-World, using `array` (instead of path as input).
+- [x] `video_demo.py`: inference YOLO-World on videos.
+- [x] `inference.ipynb`: jupyter notebook for YOLO-World.
+- [x] [Google Colab Notebook](https://colab.research.google.com/drive/1F_7S5lSaFM06irBCZqjhbN7MpUXo6WwO?usp=sharing): We sincerely thank [Onuralp](https://github.com/onuralpszr) for sharing the [Colab Demo](https://colab.research.google.com/drive/1F_7S5lSaFM06irBCZqjhbN7MpUXo6WwO?usp=sharing), you can have a try 😊！
+
+## Acknowledgement
+
+We sincerely thank [mmyolo](https://github.com/open-mmlab/mmyolo), [mmdetection](https://github.com/open-mmlab/mmdetection), [GLIP](https://github.com/microsoft/GLIP), and [transformers](https://github.com/huggingface/transformers) for providing their wonderful code to the community!
+
+## Citations
+If you find YOLO-World is useful in your research or applications, please consider giving us a star 🌟 and citing it.
+
+```bibtex
+@inproceedings{Cheng2024YOLOWorld,
+  title={YOLO-World: Real-Time Open-Vocabulary Object Detection},
+  author={Cheng, Tianheng and Song, Lin and Ge, Yixiao and Liu, Wenyu and Wang, Xinggang and Shan, Ying},
+  booktitle={Proc. IEEE Conf. Computer Vision and Pattern Recognition (CVPR)},
+  year={2024}
+}
+```
+
+## Licence
+YOLO-World is under the GPL-v3 Licence and is supported for commercial usage. If you need a commercial license for YOLO-World, please feel free to contact us.

YOLO-World/configs/finetune_coco/README.md

@@ -0,0 +1,29 @@
+## Fine-tune YOLO-World on MS-COCO
+
+
+### Updates
+
+1. [2024-3-27]: Considering that fine-tuning YOLO-World on COCO **without `mask-refine`** obtains bad results, e.g., YOLO-World-L obtains 48.6 AP without `mask-refine` compared to 53.3 AP with `mask-refine`, we rethink the training process and explore new training schemes for fine-tuning without `mask-refine`.
+BTW, the COCO fine-tuning results are updated with higher performance (with `mask-refine`)!
+
+
+### COCO Results and Checkpoints
+
+**NOTE:**
+1. AP<sup>ZS</sup>: AP evaluated in the zero-shot setting (w/o fine-tuning on COCO dataset).
+2. `mask-refine`: refine the box annotations with masks, and add `CopyPaste` augmentation during training.
+
+| model | Schedule | `mask-refine` | efficient neck | AP<sup>ZS</sup>|  AP | AP<sub>50</sub> | AP<sub>75</sub> | weights | log |
+| :---- | :-------: | :----------: |:-------------: | :------------: | :-: | :--------------:| :-------------: |:------: | :-: |
+| [YOLO-World-v2-S](./yolo_world_v2_s_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py) | AdamW, 2e-4, 80e | ✔️  | ✖️ | 37.5 | 46.1 | 62.0 | 49.9 | [HF Checkpoints](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_s_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco_ep80-492dc329.pth) | [log](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_s_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco_20240327_110411.log) |
+| [YOLO-World-v2-M](./yolo_world_v2_m_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py) | AdamW, 2e-4, 80e | ✔️  | ✖️ | 42.8 | 51.0 | 67.5 | 55.2 | [HF Checkpoints](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_m_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco_ep80-69c27ac7.pth) | [log](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_m_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco_20240327_110411.log) |
+| [YOLO-World-v2-L](./yolo_world_v2_l_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py) | AdamW, 2e-4, 80e | ✔️  | ✖️ | 45.1 | 53.9 | 70.9 | 58.8 | [HF Checkpoints](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_l_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco_ep80-81c701ee.pth) | [log](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_l_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco_20240326_160313.log) |
+| [YOLO-World-v2-X](./yolo_world_v2_x_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py) | AdamW, 2e-4, 80e | ✔️  | ✖️ | 46.8 | 54.7 | 71.6 | 59.6 | [HF Checkpoints](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_x_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco_ep80-76bc0cbd.pth) | [log](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_x_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco_20240322_181232.log) |
+| [YOLO-World-v2-L](./yolo_world_v2_l_vlpan_bn_sgd_1e-3_40e_8gpus_finetune_coco.py) 🔥 | SGD, 1e-3, 40e | ✖️  | ✖️ | 45.1 | 52.8 | 69.5 | 57.8 | [HF Checkpoints](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_l_vlpan_bn_sgd_1e-3_40e_8gpus_finetune_coco_ep80-e1288152.pth) | [log](https://huggingface.co/wondervictor/YOLO-World/blob/main/yolo_world_v2_l_vlpan_bn_sgd_1e-3_40e_8gpus_finetuning_coco_20240327_014902.log) |
+
+
+### Reparameterized Training
+
+| model | Schedule | `mask-refine` | efficient neck | AP<sup>ZS</sup>|  AP | AP<sub>50</sub> | AP<sub>75</sub> | weights | log |
+| :---- | :-------: | :----------: |:-------------: | :------------: | :-: | :--------------:| :-------------: |:------: | :-: |
+| [YOLO-World-v2-S](./yolo_world_v2_s_rep_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py) | AdamW, 2e-4, 80e | ✔️  | ✖️ | 37.5 | 46.3 | 62.8 | 50.4 | [HF Checkpoints]() | [log]() | 

YOLO-World/configs/finetune_coco/yolo_world_l_dual_vlpan_2e-4_80e_8gpus_finetune_coco.py

@@ -0,0 +1,179 @@
+_base_ = (
+    '../../third_party/mmyolo/configs/yolov8/'
+    'yolov8_l_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(
+    imports=['yolo_world'],
+    allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from='pretrained_models/yolo_world_l_clip_base_dual_vlpan_2e-3adamw_32xb16_100e_o365_goldg_train_pretrained-0e566235.pth'
+persistent_workers = False
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name='openai/clip-vit-base-patch32',
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldDualPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv'),
+              text_enhancder=dict(type='ImagePoolingAttentionModule',
+                                  embed_channels=256,
+                                  num_heads=8)),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(
+        type='MultiModalMosaic',
+        img_scale=_base_.img_scale,
+        pad_val=114.0,
+        pre_transform=_base_.pre_transform),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale,
+                             1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114))
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    *mosaic_affine_transform,
+    dict(
+        type='YOLOv5MultiModalMixUp',
+        prob=_base_.mixup_prob,
+        pre_transform=[*_base_.pre_transform,
+                       *mosaic_affine_transform]),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+train_pipeline_stage2 = [
+    *_base_.train_pipeline_stage2[:-1],
+    *text_transform
+]
+coco_train_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_train2017.json',
+        data_prefix=dict(img='train2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=train_pipeline)
+
+train_dataloader = dict(
+    persistent_workers=persistent_workers,
+    batch_size=train_batch_size_per_gpu,
+    collate_fn=dict(type='yolow_collate'),
+    dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_val2017.json',
+        data_prefix=dict(img='val2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(
+    param_scheduler=dict(
+        scheduler_type='linear',
+        lr_factor=0.01,
+        max_epochs=max_epochs),
+    checkpoint=dict(
+        max_keep_ckpts=-1,
+        save_best=None,
+        interval=save_epoch_intervals))
+custom_hooks = [
+    dict(
+        type='EMAHook',
+        ema_type='ExpMomentumEMA',
+        momentum=0.0001,
+        update_buffers=True,
+        strict_load=False,
+        priority=49),
+    dict(
+        type='mmdet.PipelineSwitchHook',
+        switch_epoch=max_epochs - close_mosaic_epochs,
+        switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(
+    max_epochs=max_epochs,
+    val_interval=5,
+    dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                        _base_.val_interval_stage2)])
+optim_wrapper = dict(
+    optimizer=dict(
+        _delete_=True,
+        type='AdamW',
+        lr=base_lr,
+        weight_decay=weight_decay,
+        batch_size_per_gpu=train_batch_size_per_gpu),
+    paramwise_cfg=dict(
+        custom_keys={'backbone.text_model': dict(lr_mult=0.01),
+                     'logit_scale': dict(weight_decay=0.0)}),
+    constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(
+    _delete_=True,
+    type='mmdet.CocoMetric',
+    proposal_nums=(100, 1, 10),
+    ann_file='data/coco/annotations/instances_val2017.json',
+    metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_l_dual_vlpan_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,181 @@
+_base_ = (
+    '../../third_party/mmyolo/configs/yolov8/'
+    'yolov8_l_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(
+    imports=['yolo_world'],
+    allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from='pretrained_models/yolo_world_l_clip_base_dual_vlpan_2e-3adamw_32xb16_100e_o365_goldg_train_pretrained-0e566235.pth'
+persistent_workers = False
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name='openai/clip-vit-base-patch32',
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldDualPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv'),
+              text_enhancder=dict(type='ImagePoolingAttentionModule',
+                                  embed_channels=256,
+                                  num_heads=8)),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(
+        type='MultiModalMosaic',
+        img_scale=_base_.img_scale,
+        pad_val=114.0,
+        pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=_base_.copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale,
+                             1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    *mosaic_affine_transform,
+    dict(
+        type='YOLOv5MultiModalMixUp',
+        prob=_base_.mixup_prob,
+        pre_transform=[*_base_.pre_transform,
+                       *mosaic_affine_transform]),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+train_pipeline_stage2 = [
+    *_base_.train_pipeline_stage2[:-1],
+    *text_transform
+]
+coco_train_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_train2017.json',
+        data_prefix=dict(img='train2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=train_pipeline)
+
+train_dataloader = dict(
+    persistent_workers=persistent_workers,
+    batch_size=train_batch_size_per_gpu,
+    collate_fn=dict(type='yolow_collate'),
+    dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_val2017.json',
+        data_prefix=dict(img='val2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(
+    param_scheduler=dict(
+        scheduler_type='linear',
+        lr_factor=0.01,
+        max_epochs=max_epochs),
+    checkpoint=dict(
+        max_keep_ckpts=-1,
+        save_best=None,
+        interval=save_epoch_intervals))
+custom_hooks = [
+    dict(
+        type='EMAHook',
+        ema_type='ExpMomentumEMA',
+        momentum=0.0001,
+        update_buffers=True,
+        strict_load=False,
+        priority=49),
+    dict(
+        type='mmdet.PipelineSwitchHook',
+        switch_epoch=max_epochs - close_mosaic_epochs,
+        switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(
+    max_epochs=max_epochs,
+    val_interval=5,
+    dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                        _base_.val_interval_stage2)])
+optim_wrapper = dict(
+    optimizer=dict(
+        _delete_=True,
+        type='AdamW',
+        lr=base_lr,
+        weight_decay=weight_decay,
+        batch_size_per_gpu=train_batch_size_per_gpu),
+    paramwise_cfg=dict(
+        custom_keys={'backbone.text_model': dict(lr_mult=0.01),
+                     'logit_scale': dict(weight_decay=0.0)}),
+    constructor='YOLOWv5OptimizerConstructor')
+# evaluation settings
+val_evaluator = dict(
+    _delete_=True,
+    type='mmdet.CocoMetric',
+    proposal_nums=(100, 1, 10),
+    ann_file='data/coco/annotations/instances_val2017.json',
+    metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_l_efficient_neck_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,159 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_l_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'], allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from = 'pretrained_models/yolo_world_l_clip_base_dual_vlpan_2e-3adamw_32xb16_100e_o365_goldg_train_pretrained-0e566235.pth'
+# huggingface text model
+text_model_name = 'openai/clip-vit-base-patch32'
+persistent_workers = False
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='EfficientCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(type='MultiModalMosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=_base_.copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform, *mosaic_affine_transform,
+    dict(type='YOLOv5MultiModalMixUp',
+         prob=_base_.mixup_prob,
+         pre_transform=[*_base_.pre_transform, *mosaic_affine_transform]),
+    *_base_.last_transform[:-1], *text_transform
+]
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *text_transform]
+coco_train_dataset = dict(_delete_=True,
+                          type='MultiModalDataset',
+                          dataset=dict(
+                              type='YOLOv5CocoDataset',
+                              data_root='data/coco',
+                              ann_file='annotations/instances_train2017.json',
+                              data_prefix=dict(img='train2017/'),
+                              filter_cfg=dict(filter_empty_gt=False,
+                                              min_size=32)),
+                          class_text_path='data/texts/coco_class_texts.json',
+                          pipeline=train_pipeline)
+
+train_dataloader = dict(persistent_workers=persistent_workers,
+                        batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5CocoDataset',
+                 data_root='data/coco',
+                 ann_file='annotations/instances_val2017.json',
+                 data_prefix=dict(img='val2017/'),
+                 filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(param_scheduler=dict(scheduler_type='linear',
+                                          lr_factor=0.01,
+                                          max_epochs=max_epochs),
+                     checkpoint=dict(max_keep_ckpts=-1,
+                                     save_best=None,
+                                     interval=save_epoch_intervals))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=5,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(
+    optimizer=dict(
+        _delete_=True,
+        type='AdamW',
+        lr=base_lr,
+        weight_decay=weight_decay,
+        batch_size_per_gpu=train_batch_size_per_gpu),
+    paramwise_cfg=dict(
+        custom_keys={'backbone.text_model': dict(lr_mult=0.01),
+                     'logit_scale': dict(weight_decay=0.0)}),
+    constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(_delete_=True,
+                     type='mmdet.CocoMetric',
+                     proposal_nums=(100, 1, 10),
+                     ann_file='data/coco/annotations/instances_val2017.json',
+                     metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_l_efficient_neck_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,182 @@
+_base_ = (
+    '../../third_party/mmyolo/configs/yolov8/'
+    'yolov8_l_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(
+    imports=['yolo_world'],
+    allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from = 'pretrained_models/yolo_world_l_clip_t2i_bn_2e-3adamw_32xb16-100e_obj365v1_goldg_cc3mlite_train-ca93cd1f.pth'
+text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+persistent_workers = False
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='EfficientCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(
+        type='MultiModalMosaic',
+        img_scale=_base_.img_scale,
+        pad_val=114.0,
+        pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=_base_.copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale,
+                             1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    *mosaic_affine_transform,
+    dict(
+        type='YOLOv5MultiModalMixUp',
+        prob=_base_.mixup_prob,
+        pre_transform=[*_base_.pre_transform,
+                       *mosaic_affine_transform]),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+train_pipeline_stage2 = [
+    *_base_.train_pipeline_stage2[:-1],
+    *text_transform
+]
+coco_train_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_train2017.json',
+        data_prefix=dict(img='train2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=train_pipeline)
+
+train_dataloader = dict(
+    persistent_workers=persistent_workers,
+    batch_size=train_batch_size_per_gpu,
+    collate_fn=dict(type='yolow_collate'),
+    dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_val2017.json',
+        data_prefix=dict(img='val2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(
+    param_scheduler=dict(
+        scheduler_type='linear',
+        lr_factor=0.01,
+        max_epochs=max_epochs),
+    checkpoint=dict(
+        max_keep_ckpts=-1,
+        save_best=None,
+        interval=save_epoch_intervals))
+custom_hooks = [
+    dict(
+        type='EMAHook',
+        ema_type='ExpMomentumEMA',
+        momentum=0.0001,
+        update_buffers=True,
+        strict_load=False,
+        priority=49),
+    dict(
+        type='mmdet.PipelineSwitchHook',
+        switch_epoch=max_epochs - close_mosaic_epochs,
+        switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(
+    max_epochs=max_epochs,
+    val_interval=5,
+    dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                        _base_.val_interval_stage2)])
+optim_wrapper = dict(
+    optimizer=dict(
+        _delete_=True,
+        type='AdamW',
+        lr=base_lr,
+        weight_decay=weight_decay,
+        batch_size_per_gpu=train_batch_size_per_gpu),
+    paramwise_cfg=dict(
+        custom_keys={'backbone.text_model': dict(lr_mult=0.01),
+                     'logit_scale': dict(weight_decay=0.0)}),
+    constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(
+    _delete_=True,
+    type='mmdet.CocoMetric',
+    proposal_nums=(100, 1, 10),
+    ann_file='data/coco/annotations/instances_val2017.json',
+    metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_l_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,181 @@
+_base_ = (
+    '../../third_party/mmyolo/configs/yolov8/'
+    'yolov8_l_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(
+    imports=['yolo_world'],
+    allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from = 'pretrained_models/yolo_world_l_clip_t2i_bn_2e-3adamw_32xb16-100e_obj365v1_goldg_cc3mlite_train-ca93cd1f.pth'
+text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+persistent_workers = False
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(
+        type='MultiModalMosaic',
+        img_scale=_base_.img_scale,
+        pad_val=114.0,
+        pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=_base_.copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale,
+                             1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    *mosaic_affine_transform,
+    dict(
+        type='YOLOv5MultiModalMixUp',
+        prob=_base_.mixup_prob,
+        pre_transform=[*_base_.pre_transform,
+                       *mosaic_affine_transform]),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+train_pipeline_stage2 = [
+    *_base_.train_pipeline_stage2[:-1],
+    *text_transform
+]
+coco_train_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_train2017.json',
+        data_prefix=dict(img='train2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=train_pipeline)
+
+train_dataloader = dict(
+    persistent_workers=persistent_workers,
+    batch_size=train_batch_size_per_gpu,
+    collate_fn=dict(type='yolow_collate'),
+    dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_val2017.json',
+        data_prefix=dict(img='val2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(
+    param_scheduler=dict(
+        scheduler_type='linear',
+        lr_factor=0.01,
+        max_epochs=max_epochs),
+    checkpoint=dict(
+        max_keep_ckpts=-1,
+        save_best=None,
+        interval=save_epoch_intervals))
+custom_hooks = [
+    dict(
+        type='EMAHook',
+        ema_type='ExpMomentumEMA',
+        momentum=0.0001,
+        update_buffers=True,
+        strict_load=False,
+        priority=49),
+    dict(
+        type='mmdet.PipelineSwitchHook',
+        switch_epoch=max_epochs - close_mosaic_epochs,
+        switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(
+    max_epochs=max_epochs,
+    val_interval=5,
+    dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                        _base_.val_interval_stage2)])
+optim_wrapper = dict(
+    optimizer=dict(
+        _delete_=True,
+        type='AdamW',
+        lr=base_lr,
+        weight_decay=weight_decay,
+        batch_size_per_gpu=train_batch_size_per_gpu),
+    paramwise_cfg=dict(
+        custom_keys={'backbone.text_model': dict(lr_mult=0.01),
+                     'logit_scale': dict(weight_decay=0.0)}),
+    constructor='YOLOWv5OptimizerConstructor')
+# evaluation settings
+val_evaluator = dict(
+    _delete_=True,
+    type='mmdet.CocoMetric',
+    proposal_nums=(100, 1, 10),
+    ann_file='data/coco/annotations/instances_val2017.json',
+    metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_l_vlpan_bn_sgd_1e-3_40e_8gpus_finetune_coco.py

@@ -0,0 +1,160 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_l_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'], allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 40  # Maximum training epochs
+close_mosaic_epochs = 30
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 1e-3
+weight_decay = 0.0005
+train_batch_size_per_gpu = 16
+load_from = 'pretrained_models/yolo_world_l_clip_t2i_bn_2e-3adamw_32xb16-100e_obj365v1_goldg_cc3mlite_train-ca93cd1f.pth'
+text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+persistent_workers = False
+
+# model settings
+model = dict(type='YOLOWorldDetector',
+             mm_neck=True,
+             num_train_classes=num_training_classes,
+             num_test_classes=num_classes,
+             data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+             backbone=dict(_delete_=True,
+                           type='MultiModalYOLOBackbone',
+                           image_model={{_base_.model.backbone}},
+                           text_model=dict(type='HuggingCLIPLanguageBackbone',
+                                           model_name=text_model_name,
+                                           frozen_modules=['all'])),
+             neck=dict(type='YOLOWorldPAFPN',
+                       guide_channels=text_channels,
+                       embed_channels=neck_embed_channels,
+                       num_heads=neck_num_heads,
+                       block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+             bbox_head=dict(type='YOLOWorldHead',
+                            head_module=dict(
+                                type='YOLOWorldHeadModule',
+                                use_bn_head=True,
+                                embed_dims=text_channels,
+                                num_classes=num_training_classes)),
+             train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(type='MultiModalMosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114))
+]
+
+train_pipeline = [
+    *_base_.pre_transform, *mosaic_affine_transform,
+    dict(type='YOLOv5MultiModalMixUp',
+         prob=_base_.mixup_prob,
+         pre_transform=[*_base_.pre_transform, *mosaic_affine_transform]),
+    *_base_.last_transform[:-1], *text_transform
+]
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *text_transform]
+
+coco_train_dataset = dict(_delete_=True,
+                          type='MultiModalDataset',
+                          dataset=dict(
+                              type='YOLOv5CocoDataset',
+                              data_root='data/coco',
+                              ann_file='annotations/instances_train2017.json',
+                              data_prefix=dict(img='train2017/'),
+                              filter_cfg=dict(filter_empty_gt=False,
+                                              min_size=32)),
+                          class_text_path='data/texts/coco_class_texts.json',
+                          pipeline=train_pipeline)
+
+train_dataloader = dict(persistent_workers=persistent_workers,
+                        batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5CocoDataset',
+                 data_root='data/coco',
+                 ann_file='annotations/instances_val2017.json',
+                 data_prefix=dict(img='val2017/'),
+                 filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(param_scheduler=dict(scheduler_type='linear',
+                                          lr_factor=0.01,
+                                          max_epochs=max_epochs),
+                     checkpoint=dict(max_keep_ckpts=-1,
+                                     save_best=None,
+                                     interval=save_epoch_intervals))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=5,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='SGD',
+    lr=base_lr,
+    momentum=0.937,
+    nesterov=True,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(
+                         custom_keys={
+                             'backbone.text_model': dict(lr_mult=0.01),
+                             'logit_scale': dict(weight_decay=0.0)
+                         }),
+                     constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(_delete_=True,
+                     type='mmdet.CocoMetric',
+                     proposal_nums=(100, 1, 10),
+                     ann_file='data/coco/annotations/instances_val2017.json',
+                     metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_l_vlpan_bn_sgd_1e-3_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,161 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_l_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'], allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 1e-3
+weight_decay = 0.0005
+train_batch_size_per_gpu = 16
+load_from = 'pretrained_models/yolo_world_l_clip_t2i_bn_2e-3adamw_32xb16-100e_obj365v1_goldg_cc3mlite_train-ca93cd1f.pth'
+text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+persistent_workers = False
+
+# model settings
+model = dict(type='YOLOWorldDetector',
+             mm_neck=True,
+             num_train_classes=num_training_classes,
+             num_test_classes=num_classes,
+             data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+             backbone=dict(_delete_=True,
+                           type='MultiModalYOLOBackbone',
+                           image_model={{_base_.model.backbone}},
+                           text_model=dict(type='HuggingCLIPLanguageBackbone',
+                                           model_name=text_model_name,
+                                           frozen_modules=['all'])),
+             neck=dict(type='YOLOWorldPAFPN',
+                       guide_channels=text_channels,
+                       embed_channels=neck_embed_channels,
+                       num_heads=neck_num_heads,
+                       block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+             bbox_head=dict(type='YOLOWorldHead',
+                            head_module=dict(
+                                type='YOLOWorldHeadModule',
+                                use_bn_head=True,
+                                embed_dims=text_channels,
+                                num_classes=num_training_classes)),
+             train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(type='MultiModalMosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=_base_.copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform, *mosaic_affine_transform,
+    dict(type='YOLOv5MultiModalMixUp',
+         prob=_base_.mixup_prob,
+         pre_transform=[*_base_.pre_transform, *mosaic_affine_transform]),
+    *_base_.last_transform[:-1], *text_transform
+]
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *text_transform]
+coco_train_dataset = dict(_delete_=True,
+                          type='MultiModalDataset',
+                          dataset=dict(
+                              type='YOLOv5CocoDataset',
+                              data_root='data/coco',
+                              ann_file='annotations/instances_train2017.json',
+                              data_prefix=dict(img='train2017/'),
+                              filter_cfg=dict(filter_empty_gt=False,
+                                              min_size=32)),
+                          class_text_path='data/texts/coco_class_texts.json',
+                          pipeline=train_pipeline)
+
+train_dataloader = dict(persistent_workers=persistent_workers,
+                        batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5CocoDataset',
+                 data_root='data/coco',
+                 ann_file='annotations/instances_val2017.json',
+                 data_prefix=dict(img='val2017/'),
+                 filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(param_scheduler=dict(scheduler_type='linear',
+                                          lr_factor=0.01,
+                                          max_epochs=max_epochs),
+                     checkpoint=dict(max_keep_ckpts=-1,
+                                     save_best=None,
+                                     interval=save_epoch_intervals))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=5,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='SGD',
+    lr=base_lr,
+    momentum=0.937,
+    nesterov=True,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(
+                         custom_keys={
+                             'backbone.text_model': dict(lr_mult=0.01),
+                             'logit_scale': dict(weight_decay=0.0)
+                         }),
+                     constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(_delete_=True,
+                     type='mmdet.CocoMetric',
+                     proposal_nums=(100, 1, 10),
+                     ann_file='data/coco/annotations/instances_val2017.json',
+                     metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_m_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,182 @@
+_base_ = (
+    '../../third_party/mmyolo/configs/yolov8/'
+    'yolov8_m_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(
+    imports=['yolo_world'],
+    allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from = 'pretrained_models/yolo_world_m_clip_t2i_bn_2e-3adamw_32xb16-100e_obj365v1_goldg_train-c6237d5b.pth'
+# text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+persistent_workers = False
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(
+        type='MultiModalMosaic',
+        img_scale=_base_.img_scale,
+        pad_val=114.0,
+        pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=_base_.copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale,
+                             1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    *mosaic_affine_transform,
+    dict(
+        type='YOLOv5MultiModalMixUp',
+        prob=_base_.mixup_prob,
+        pre_transform=[*_base_.pre_transform,
+                       *mosaic_affine_transform]),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+train_pipeline_stage2 = [
+    *_base_.train_pipeline_stage2[:-1],
+    *text_transform
+]
+coco_train_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_train2017.json',
+        data_prefix=dict(img='train2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=train_pipeline)
+
+train_dataloader = dict(
+    persistent_workers=persistent_workers,
+    batch_size=train_batch_size_per_gpu,
+    collate_fn=dict(type='yolow_collate'),
+    dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_val2017.json',
+        data_prefix=dict(img='val2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(
+    param_scheduler=dict(
+        scheduler_type='linear',
+        lr_factor=0.01,
+        max_epochs=max_epochs),
+    checkpoint=dict(
+        max_keep_ckpts=-1,
+        save_best=None,
+        interval=save_epoch_intervals))
+custom_hooks = [
+    dict(
+        type='EMAHook',
+        ema_type='ExpMomentumEMA',
+        momentum=0.0001,
+        update_buffers=True,
+        strict_load=False,
+        priority=49),
+    dict(
+        type='mmdet.PipelineSwitchHook',
+        switch_epoch=max_epochs - close_mosaic_epochs,
+        switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(
+    max_epochs=max_epochs,
+    val_interval=5,
+    dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                        _base_.val_interval_stage2)])
+optim_wrapper = dict(
+    optimizer=dict(
+        _delete_=True,
+        type='AdamW',
+        lr=base_lr,
+        weight_decay=weight_decay,
+        batch_size_per_gpu=train_batch_size_per_gpu),
+    paramwise_cfg=dict(
+        custom_keys={'backbone.text_model': dict(lr_mult=0.01),
+                     'logit_scale': dict(weight_decay=0.0)}),
+    constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(
+    _delete_=True,
+    type='mmdet.CocoMetric',
+    proposal_nums=(100, 1, 10),
+    ann_file='data/coco/annotations/instances_val2017.json',
+    metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_s_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,145 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_s_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'], allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from = '../FastDet/output_models/pretrain_yolow-v8_s_clipv2_frozen_te_noprompt_t2i_bn_2e-3adamw_scale_lr_wd_32xb16-100e_obj365v1_goldg_cc3mram250k_train_lviseval-e3592307_rep_conv.pth'
+persistent_workers = False
+mixup_prob = 0.15
+copypaste_prob = 0.3
+
+# model settings
+model = dict(type='SimpleYOLOWorldDetector',
+             mm_neck=True,
+             num_train_classes=num_classes,
+             num_test_classes=num_classes,
+             reparameterized=True,
+             data_preprocessor=dict(type='YOLOv5DetDataPreprocessor'),
+             backbone=dict(_delete_=True,
+                           type='MultiModalYOLOBackbone',
+                           text_model=None,
+                           image_model={{_base_.model.backbone}},
+                           with_text_model=False),
+             neck=dict(type='YOLOWorldPAFPN',
+                       guide_channels=text_channels,
+                       embed_channels=neck_embed_channels,
+                       num_heads=neck_num_heads,
+                       block_cfg=dict(type='EfficientCSPLayerWithTwoConv')),
+             bbox_head=dict(head_module=dict(type='RepYOLOWorldHeadModule',
+                                             embed_dims=text_channels,
+                                             num_guide=num_classes,
+                                             num_classes=num_classes)),
+             train_cfg=dict(assigner=dict(num_classes=num_classes)))
+
+# dataset settings
+final_transform = [
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction'))
+]
+mosaic_affine_transform = [
+    dict(type='Mosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform, *mosaic_affine_transform,
+    dict(type='YOLOv5MixUp',
+         prob=mixup_prob,
+         pre_transform=[*_base_.pre_transform, *mosaic_affine_transform]),
+    *_base_.last_transform[:-1], *final_transform
+]
+
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *final_transform]
+
+coco_train_dataset = dict(type='YOLOv5CocoDataset',
+                          data_root='data/coco',
+                          ann_file='annotations/instances_train2017.json',
+                          data_prefix=dict(img='train2017/'),
+                          filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                          pipeline=train_pipeline)
+
+train_dataloader = dict(persistent_workers=persistent_workers,
+                        batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=coco_train_dataset)
+
+train_dataloader = dict(persistent_workers=persistent_workers,
+                        batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param'))
+]
+coco_val_dataset = dict(type='YOLOv5CocoDataset',
+                        data_root='data/coco',
+                        ann_file='annotations/instances_val2017.json',
+                        data_prefix=dict(img='val2017/'),
+                        filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                        pipeline=test_pipeline)
+
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(param_scheduler=dict(scheduler_type='linear',
+                                          lr_factor=0.01,
+                                          max_epochs=max_epochs),
+                     checkpoint=dict(max_keep_ckpts=-1,
+                                     save_best=None,
+                                     interval=save_epoch_intervals))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=5,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(_delete_=True,
+                     type='mmdet.CocoMetric',
+                     proposal_nums=(100, 1, 10),
+                     ann_file='data/coco/annotations/instances_val2017.json',
+                     metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_s_rep_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,146 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_s_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'], allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from = '../FastDet/output_models/yolo_world_s_clip_t2i_bn_2e-3adamw_32xb16-100e_obj365v1_goldg_train-55b943ea_rep_conv.pth'
+persistent_workers = False
+mixup_prob = 0.15
+copypaste_prob = 0.3
+
+# model settings
+model = dict(type='SimpleYOLOWorldDetector',
+             mm_neck=True,
+             num_train_classes=num_classes,
+             num_test_classes=num_classes,
+             reparameterized=True,
+             data_preprocessor=dict(type='YOLOv5DetDataPreprocessor'),
+             backbone=dict(_delete_=True,
+                           type='MultiModalYOLOBackbone',
+                           text_model=None,
+                           image_model={{_base_.model.backbone}},
+                           with_text_model=False),
+             neck=dict(type='YOLOWorldPAFPN',
+                       guide_channels=num_classes,
+                       embed_channels=neck_embed_channels,
+                       num_heads=neck_num_heads,
+                       block_cfg=dict(type='RepConvMaxSigmoidCSPLayerWithTwoConv',
+                                      guide_channels=num_classes)),
+             bbox_head=dict(head_module=dict(type='RepYOLOWorldHeadModule',
+                                             embed_dims=text_channels,
+                                             num_guide=num_classes,
+                                             num_classes=num_classes)),
+             train_cfg=dict(assigner=dict(num_classes=num_classes)))
+
+# dataset settings
+final_transform = [
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction'))
+]
+mosaic_affine_transform = [
+    dict(type='Mosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform, *mosaic_affine_transform,
+    dict(type='YOLOv5MixUp',
+         prob=mixup_prob,
+         pre_transform=[*_base_.pre_transform, *mosaic_affine_transform]),
+    *_base_.last_transform[:-1], *final_transform
+]
+
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *final_transform]
+
+coco_train_dataset = dict(type='YOLOv5CocoDataset',
+                          data_root='data/coco',
+                          ann_file='annotations/instances_train2017.json',
+                          data_prefix=dict(img='train2017/'),
+                          filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                          pipeline=train_pipeline)
+
+train_dataloader = dict(persistent_workers=persistent_workers,
+                        batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=coco_train_dataset)
+
+train_dataloader = dict(persistent_workers=persistent_workers,
+                        batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param'))
+]
+coco_val_dataset = dict(type='YOLOv5CocoDataset',
+                        data_root='data/coco',
+                        ann_file='annotations/instances_val2017.json',
+                        data_prefix=dict(img='val2017/'),
+                        filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                        pipeline=test_pipeline)
+
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(param_scheduler=dict(scheduler_type='linear',
+                                          lr_factor=0.01,
+                                          max_epochs=max_epochs),
+                     checkpoint=dict(max_keep_ckpts=-1,
+                                     save_best=None,
+                                     interval=save_epoch_intervals))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=5,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(_delete_=True,
+                     type='mmdet.CocoMetric',
+                     proposal_nums=(100, 1, 10),
+                     ann_file='data/coco/annotations/instances_val2017.json',
+                     metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_s_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,184 @@
+_base_ = (
+    '../../third_party/mmyolo/configs/yolov8/'
+    'yolov8_s_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(
+    imports=['yolo_world'],
+    allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from = 'pretrained_models/yolo_world_s_clip_t2i_bn_2e-3adamw_32xb16-100e_obj365v1_goldg_train-55b943ea.pth'
+text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+persistent_workers = False
+mixup_prob = 0.15
+copypaste_prob = 0.3
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(
+        type='MultiModalMosaic',
+        img_scale=_base_.img_scale,
+        pad_val=114.0,
+        pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale,
+                             1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    *mosaic_affine_transform,
+    dict(
+        type='YOLOv5MultiModalMixUp',
+        prob=mixup_prob,
+        pre_transform=[*_base_.pre_transform,
+                       *mosaic_affine_transform]),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+train_pipeline_stage2 = [
+    *_base_.train_pipeline_stage2[:-1],
+    *text_transform
+]
+coco_train_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_train2017.json',
+        data_prefix=dict(img='train2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=train_pipeline)
+
+train_dataloader = dict(
+    persistent_workers=persistent_workers,
+    batch_size=train_batch_size_per_gpu,
+    collate_fn=dict(type='yolow_collate'),
+    dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_val2017.json',
+        data_prefix=dict(img='val2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(
+    param_scheduler=dict(
+        scheduler_type='linear',
+        lr_factor=0.01,
+        max_epochs=max_epochs),
+    checkpoint=dict(
+        max_keep_ckpts=-1,
+        save_best=None,
+        interval=save_epoch_intervals))
+custom_hooks = [
+    dict(
+        type='EMAHook',
+        ema_type='ExpMomentumEMA',
+        momentum=0.0001,
+        update_buffers=True,
+        strict_load=False,
+        priority=49),
+    dict(
+        type='mmdet.PipelineSwitchHook',
+        switch_epoch=max_epochs - close_mosaic_epochs,
+        switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(
+    max_epochs=max_epochs,
+    val_interval=5,
+    dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                        _base_.val_interval_stage2)])
+optim_wrapper = dict(
+    optimizer=dict(
+        _delete_=True,
+        type='AdamW',
+        lr=base_lr,
+        weight_decay=weight_decay,
+        batch_size_per_gpu=train_batch_size_per_gpu),
+    paramwise_cfg=dict(
+        custom_keys={'backbone.text_model': dict(lr_mult=0.01),
+                     'logit_scale': dict(weight_decay=0.0)}),
+    constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(
+    _delete_=True,
+    type='mmdet.CocoMetric',
+    proposal_nums=(100, 1, 10),
+    ann_file='data/coco/annotations/instances_val2017.json',
+    metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_x_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,183 @@
+_base_ = (
+    '../../third_party/mmyolo/configs/yolov8/'
+    'yolov8_x_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(
+    imports=['yolo_world'],
+    allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from = 'pretrained_models/yolo_world_x_clip_t2i_bn_2e-3adamw_32xb16-100e_obj365v1_goldg_cc250k_train_lviseval-8698fbfa.pth'
+text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+persistent_workers = False
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(
+        type='MultiModalMosaic',
+        img_scale=_base_.img_scale,
+        pad_val=114.0,
+        pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=_base_.copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale,
+                             1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    *mosaic_affine_transform,
+    dict(
+        type='YOLOv5MultiModalMixUp',
+        prob=_base_.mixup_prob,
+        pre_transform=[*_base_.pre_transform,
+                       *mosaic_affine_transform]),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+train_pipeline_stage2 = [
+    *_base_.train_pipeline_stage2[:-1],
+    *text_transform
+]
+coco_train_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_train2017.json',
+        data_prefix=dict(img='train2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=train_pipeline)
+
+train_dataloader = dict(
+    persistent_workers=persistent_workers,
+    batch_size=train_batch_size_per_gpu,
+    collate_fn=dict(type='yolow_collate'),
+    dataset=coco_train_dataset)
+
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor', 'pad_param', 'texts'))
+]
+
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_val2017.json',
+        data_prefix=dict(img='val2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(
+    param_scheduler=dict(
+        scheduler_type='linear',
+        lr_factor=0.01,
+        max_epochs=max_epochs),
+    checkpoint=dict(
+        max_keep_ckpts=-1,
+        save_best=None,
+        interval=save_epoch_intervals))
+custom_hooks = [
+    dict(
+        type='EMAHook',
+        ema_type='ExpMomentumEMA',
+        momentum=0.0001,
+        update_buffers=True,
+        strict_load=False,
+        priority=49),
+    dict(
+        type='mmdet.PipelineSwitchHook',
+        switch_epoch=max_epochs - close_mosaic_epochs,
+        switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(
+    max_epochs=max_epochs,
+    val_interval=5,
+    dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                        _base_.val_interval_stage2)])
+optim_wrapper = dict(
+    optimizer=dict(
+        _delete_=True,
+        type='AdamW',
+        lr=base_lr,
+        weight_decay=weight_decay,
+        batch_size_per_gpu=train_batch_size_per_gpu),
+    paramwise_cfg=dict(
+        custom_keys={'backbone.text_model': dict(lr_mult=0.01),
+                     'logit_scale': dict(weight_decay=0.0)}),
+    constructor='YOLOWv5OptimizerConstructor')
+# evaluation settings
+val_evaluator = dict(
+    _delete_=True,
+    type='mmdet.CocoMetric',
+    proposal_nums=(100, 1, 10),
+    ann_file='data/coco/annotations/instances_val2017.json',
+    metric='bbox')

YOLO-World/configs/finetune_coco/yolo_world_v2_xl_vlpan_bn_2e-4_80e_8gpus_mask-refine_finetune_coco.py

@@ -0,0 +1,173 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_x_mask-refine_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'], allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+persistent_workers = False
+
+# scaling model from X to XL
+deepen_factor = 1.0
+widen_factor = 1.5
+
+backbone = _base_.model.backbone
+backbone.update(deepen_factor=deepen_factor, widen_factor=widen_factor)
+
+# model settings
+model = dict(type='YOLOWorldDetector',
+             mm_neck=True,
+             num_train_classes=num_training_classes,
+             num_test_classes=num_classes,
+             data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+             backbone=dict(_delete_=True,
+                           type='MultiModalYOLOBackbone',
+                           image_model=backbone,
+                           text_model=dict(type='HuggingCLIPLanguageBackbone',
+                                           model_name=text_model_name,
+                                           frozen_modules=['all'])),
+             neck=dict(type='YOLOWorldPAFPN',
+                       deepen_factor=deepen_factor,
+                       widen_factor=widen_factor,
+                       guide_channels=text_channels,
+                       embed_channels=neck_embed_channels,
+                       num_heads=neck_num_heads,
+                       block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+             bbox_head=dict(type='YOLOWorldHead',
+                            head_module=dict(
+                                type='YOLOWorldHeadModule',
+                                widen_factor=widen_factor,
+                                use_bn_head=True,
+                                embed_dims=text_channels,
+                                num_classes=num_training_classes)),
+             train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+mosaic_affine_transform = [
+    dict(type='MultiModalMosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(type='YOLOv5CopyPaste', prob=_base_.copypaste_prob),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        max_aspect_ratio=100.,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        # img_scale is (width, height)
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114),
+        min_area_ratio=_base_.min_area_ratio,
+        use_mask_refine=_base_.use_mask2refine)
+]
+train_pipeline = [
+    *_base_.pre_transform, *mosaic_affine_transform,
+    dict(type='YOLOv5MultiModalMixUp',
+         prob=_base_.mixup_prob,
+         pre_transform=[*_base_.pre_transform, *mosaic_affine_transform]),
+    *_base_.last_transform[:-1], *text_transform
+]
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *text_transform]
+coco_train_dataset = dict(_delete_=True,
+                          type='MultiModalDataset',
+                          dataset=dict(
+                              type='YOLOv5CocoDataset',
+                              data_root='data/coco',
+                              ann_file='annotations/instances_train2017.json',
+                              data_prefix=dict(img='train2017/'),
+                              filter_cfg=dict(filter_empty_gt=False,
+                                              min_size=32)),
+                          class_text_path='data/texts/coco_class_texts.json',
+                          pipeline=train_pipeline)
+
+train_dataloader = dict(persistent_workers=persistent_workers,
+                        batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=coco_train_dataset)
+
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5CocoDataset',
+                 data_root='data/coco',
+                 ann_file='annotations/instances_val2017.json',
+                 data_prefix=dict(img='val2017/'),
+                 filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(param_scheduler=dict(scheduler_type='linear',
+                                          lr_factor=0.01,
+                                          max_epochs=max_epochs),
+                     checkpoint=dict(max_keep_ckpts=-1,
+                                     save_best=None,
+                                     interval=save_epoch_intervals))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=5,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(bias_decay_mult=0.0,
+                                        norm_decay_mult=0.0,
+                                        custom_keys={
+                                            'backbone.text_model':
+                                            dict(lr_mult=0.01),
+                                            'logit_scale':
+                                            dict(weight_decay=0.0)
+                                        }),
+                     constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(_delete_=True,
+                     type='mmdet.CocoMetric',
+                     proposal_nums=(100, 1, 10),
+                     ann_file='data/coco/annotations/instances_val2017.json',
+                     metric='bbox')

YOLO-World/configs/image_prompts/yolo_world_v2_l_vlpan_bn_2e-4_80e_8gpus_image_prompt_demo.py

@@ -0,0 +1,127 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_l_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'], allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 80
+num_training_classes = 80
+max_epochs = 80  # Maximum training epochs
+close_mosaic_epochs = 10
+save_epoch_intervals = 5
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.05
+train_batch_size_per_gpu = 16
+load_from = 'pretrained_models/yolo_world_l_clip_t2i_bn_2e-3adamw_32xb16-100e_obj365v1_goldg_cc3mlite_train-ca93cd1f.pth'
+persistent_workers = False
+text_model_name = '../pretrained_models/open-ai-clip-vit-base-patch32'
+img_scale = (800, 800)
+
+# model settings
+model = dict(type='YOLOWorldImageDetector',
+             mm_neck=True,
+             num_train_classes=num_training_classes,
+             num_test_classes=num_classes,
+             vision_model=text_model_name,
+             prompt_dim=text_channels,
+             data_preprocessor=dict(type='YOLOv5DetDataPreprocessor'),
+             backbone=dict(_delete_=True,
+                           type='MultiModalYOLOBackbone',
+                           image_model={{_base_.model.backbone}},
+                           frozen_stages=4,
+                           text_model=dict(type='HuggingCLIPLanguageBackbone',
+                                           model_name=text_model_name,
+                                           frozen_modules=['all'])),
+             neck=dict(type='YOLOWorldPAFPN',
+                       freeze_all=True,
+                       guide_channels=text_channels,
+                       embed_channels=neck_embed_channels,
+                       num_heads=neck_num_heads,
+                       block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+             bbox_head=dict(type='YOLOWorldHead',
+                            head_module=dict(
+                                type='YOLOWorldHeadModule',
+                                freeze_all=True,
+                                use_bn_head=True,
+                                embed_dims=text_channels,
+                                num_classes=num_training_classes)),
+             train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+coco_train_dataset = dict(type='YOLOv5CocoDataset',
+                          data_root='data/coco',
+                          ann_file='annotations/instances_train2017.json',
+                          data_prefix=dict(img='train2017/'),
+                          filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                          pipeline=_base_.train_pipeline)
+
+train_dataloader = dict(persistent_workers=persistent_workers,
+                        batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=coco_train_dataset)
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5CocoDataset',
+        data_root='data/coco',
+        ann_file='annotations/instances_val2017.json',
+        data_prefix=dict(img='val2017/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/coco_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+# training settings
+default_hooks = dict(param_scheduler=dict(scheduler_type='linear',
+                                          lr_factor=0.01,
+                                          max_epochs=max_epochs),
+                     checkpoint=dict(max_keep_ckpts=-1,
+                                     save_best=None,
+                                     interval=save_epoch_intervals))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=_base_.train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=5,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(
+                         custom_keys={
+                             'backbone.text_model': dict(lr_mult=0.01),
+                             'logit_scale': dict(weight_decay=0.0),
+                             'embeddings': dict(weight_decay=0.0)
+                         }),
+                     constructor='YOLOWv5OptimizerConstructor')
+
+# evaluation settings
+val_evaluator = dict(_delete_=True,
+                     type='mmdet.CocoMetric',
+                     proposal_nums=(100, 1, 10),
+                     ann_file='data/coco/annotations/instances_val2017.json',
+                     metric='bbox')

YOLO-World/configs/pretrain/yolo_world_v2_l_clip_large_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_800ft_lvis_minival.py

@@ -0,0 +1,200 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_l_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'],
+                      allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 1203
+num_training_classes = 80
+max_epochs = 100  # Maximum training epochs
+close_mosaic_epochs = 2
+save_epoch_intervals = 2
+text_channels = 768
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-3
+weight_decay = 0.0125
+train_batch_size_per_gpu = 16
+# text_model_name = '../pretrained_models/clip-vit-large-patch14-336'
+text_model_name = 'openai/clip-vit-large-patch14-336'
+img_scale = (800, 800)
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    dict(type='MultiModalMosaic',
+         img_scale=img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border=(-img_scale[0] // 2, -img_scale[1] // 2),
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform,
+]
+
+train_pipeline_stage2 = [
+    *_base_.pre_transform,
+    dict(type='YOLOv5KeepRatioResize', scale=img_scale),
+    dict(
+        type='LetterResize',
+        scale=img_scale,
+        allow_scale_up=True,
+        pad_val=dict(img=114.0)),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+
+obj365v1_train_dataset = dict(
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5Objects365V1Dataset',
+        data_root='data/objects365v1/',
+        ann_file='annotations/objects365_train.json',
+        data_prefix=dict(img='train/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/obj365v1_class_texts.json',
+    pipeline=train_pipeline)
+
+mg_train_dataset = dict(type='YOLOv5MixedGroundingDataset',
+                        data_root='data/mixed_grounding/',
+                        ann_file='annotations/final_mixed_train_no_coco.json',
+                        data_prefix=dict(img='gqa/images/'),
+                        filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                        pipeline=train_pipeline)
+
+flickr_train_dataset = dict(
+    type='YOLOv5MixedGroundingDataset',
+    data_root='data/flickr/',
+    ann_file='annotations/final_flickr_separateGT_train.json',
+    data_prefix=dict(img='full_images/'),
+    filter_cfg=dict(filter_empty_gt=True, min_size=32),
+    pipeline=train_pipeline)
+
+train_dataloader = dict(batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=dict(_delete_=True,
+                                     type='ConcatDataset',
+                                     datasets=[
+                                         obj365v1_train_dataset,
+                                         flickr_train_dataset, mg_train_dataset
+                                     ],
+                                     ignore_keys=['classes', 'palette']))
+
+test_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='YOLOv5KeepRatioResize', scale=img_scale),
+    dict(
+        type='LetterResize',
+        scale=img_scale,
+        allow_scale_up=False,
+        pad_val=dict(img=114)),
+    dict(type='LoadAnnotations', with_bbox=True, _scope_='mmdet'),
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5LVISV1Dataset',
+                 data_root='data/coco/',
+                 test_mode=True,
+                 ann_file='lvis/lvis_v1_minival_inserted_image_name.json',
+                 data_prefix=dict(img=''),
+                 batch_shapes_cfg=None),
+    class_text_path='data/texts/lvis_v1_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+
+val_evaluator = dict(type='mmdet.LVISMetric',
+                     ann_file='data/coco/lvis/lvis_v1_minival_inserted_image_name.json',
+                     metric='bbox')
+test_evaluator = val_evaluator
+
+# training settings
+default_hooks = dict(param_scheduler=dict(max_epochs=max_epochs),
+                     checkpoint=dict(interval=save_epoch_intervals,
+                                     rule='greater'))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=10,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(bias_decay_mult=0.0,
+                                        norm_decay_mult=0.0,
+                                        custom_keys={
+                                            'backbone.text_model':
+                                            dict(lr_mult=0.01),
+                                            'logit_scale':
+                                            dict(weight_decay=0.0)
+                                        }),
+                     constructor='YOLOWv5OptimizerConstructor')

YOLO-World/configs/pretrain/yolo_world_v2_l_clip_large_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py

@@ -0,0 +1,171 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_l_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'],
+                      allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 1203
+num_training_classes = 80
+max_epochs = 100  # Maximum training epochs
+close_mosaic_epochs = 2
+save_epoch_intervals = 2
+text_channels = 768
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-3
+weight_decay = 0.0125
+train_batch_size_per_gpu = 16
+# text_model_name = '../pretrained_models/clip-vit-large-patch14-336'
+text_model_name = 'openai/clip-vit-large-patch14-336'
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    dict(type='MultiModalMosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform,
+]
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *text_transform]
+obj365v1_train_dataset = dict(
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5Objects365V1Dataset',
+        data_root='data/objects365v1/',
+        ann_file='annotations/objects365_train.json',
+        data_prefix=dict(img='train/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/obj365v1_class_texts.json',
+    pipeline=train_pipeline)
+
+mg_train_dataset = dict(type='YOLOv5MixedGroundingDataset',
+                        data_root='data/mixed_grounding/',
+                        ann_file='annotations/final_mixed_train_no_coco.json',
+                        data_prefix=dict(img='gqa/images/'),
+                        filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                        pipeline=train_pipeline)
+
+flickr_train_dataset = dict(
+    type='YOLOv5MixedGroundingDataset',
+    data_root='data/flickr/',
+    ann_file='annotations/final_flickr_separateGT_train.json',
+    data_prefix=dict(img='full_images/'),
+    filter_cfg=dict(filter_empty_gt=True, min_size=32),
+    pipeline=train_pipeline)
+
+train_dataloader = dict(batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=dict(_delete_=True,
+                                     type='ConcatDataset',
+                                     datasets=[
+                                         obj365v1_train_dataset,
+                                         flickr_train_dataset, mg_train_dataset
+                                     ],
+                                     ignore_keys=['classes', 'palette']))
+
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5LVISV1Dataset',
+                 data_root='data/coco/',
+                 test_mode=True,
+                 ann_file='lvis/lvis_v1_minival_inserted_image_name.json',
+                 data_prefix=dict(img=''),
+                 batch_shapes_cfg=None),
+    class_text_path='data/texts/lvis_v1_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+
+val_evaluator = dict(type='mmdet.LVISMetric',
+                     ann_file='data/coco/lvis/lvis_v1_minival_inserted_image_name.json',
+                     metric='bbox')
+test_evaluator = val_evaluator
+
+# training settings
+default_hooks = dict(param_scheduler=dict(max_epochs=max_epochs),
+                     checkpoint=dict(interval=save_epoch_intervals,
+                                     rule='greater'))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=10,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(bias_decay_mult=0.0,
+                                        norm_decay_mult=0.0,
+                                        custom_keys={
+                                            'backbone.text_model':
+                                            dict(lr_mult=0.01),
+                                            'logit_scale':
+                                            dict(weight_decay=0.0)
+                                        }),
+                     constructor='YOLOWv5OptimizerConstructor')

YOLO-World/configs/pretrain/yolo_world_v2_l_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_1280ft_lvis_minival.py

@@ -0,0 +1,202 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_l_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'],
+                      allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 1203
+num_training_classes = 80
+max_epochs = 20  # Maximum training epochs
+close_mosaic_epochs = 2
+save_epoch_intervals = 2
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-4
+weight_decay = 0.025
+train_batch_size_per_gpu = 4
+load_from = "pretrained_models/yolo_world_v2_l_obj365v1_goldg_pretrain-a82b1fe3.pth"
+# text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+img_scale = (1280, 1280)
+
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    dict(type='MultiModalMosaic',
+         img_scale=img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border=(-img_scale[0] // 2, -img_scale[1] // 2),
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform,
+]
+
+train_pipeline_stage2 = [
+    *_base_.pre_transform,
+    dict(type='YOLOv5KeepRatioResize', scale=img_scale),
+    dict(
+        type='LetterResize',
+        scale=img_scale,
+        allow_scale_up=True,
+        pad_val=dict(img=114.0)),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+
+obj365v1_train_dataset = dict(
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5Objects365V1Dataset',
+        data_root='data/objects365v1/',
+        ann_file='annotations/objects365_train.json',
+        data_prefix=dict(img='train/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/obj365v1_class_texts.json',
+    pipeline=train_pipeline)
+
+mg_train_dataset = dict(type='YOLOv5MixedGroundingDataset',
+                        data_root='data/mixed_grounding/',
+                        ann_file='annotations/final_mixed_train_no_coco.json',
+                        data_prefix=dict(img='gqa/images/'),
+                        filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                        pipeline=train_pipeline)
+
+flickr_train_dataset = dict(
+    type='YOLOv5MixedGroundingDataset',
+    data_root='data/flickr/',
+    ann_file='annotations/final_flickr_separateGT_train.json',
+    data_prefix=dict(img='full_images/'),
+    filter_cfg=dict(filter_empty_gt=True, min_size=32),
+    pipeline=train_pipeline)
+
+train_dataloader = dict(batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=dict(_delete_=True,
+                                     type='ConcatDataset',
+                                     datasets=[
+                                         obj365v1_train_dataset,
+                                         flickr_train_dataset, mg_train_dataset
+                                     ],
+                                     ignore_keys=['classes', 'palette']))
+
+test_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='YOLOv5KeepRatioResize', scale=img_scale),
+    dict(
+        type='LetterResize',
+        scale=img_scale,
+        allow_scale_up=False,
+        pad_val=dict(img=114)),
+    dict(type='LoadAnnotations', with_bbox=True, _scope_='mmdet'),
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5LVISV1Dataset',
+                 data_root='data/coco/',
+                 test_mode=True,
+                 ann_file='lvis/lvis_v1_minival_inserted_image_name.json',
+                 data_prefix=dict(img=''),
+                 batch_shapes_cfg=None),
+    class_text_path='data/texts/lvis_v1_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+
+val_evaluator = dict(type='mmdet.LVISMetric',
+                     ann_file='data/coco/lvis/lvis_v1_minival_inserted_image_name.json',
+                     metric='bbox')
+test_evaluator = val_evaluator
+
+# training settings
+default_hooks = dict(param_scheduler=dict(max_epochs=max_epochs),
+                     checkpoint=dict(interval=save_epoch_intervals,
+                                     rule='greater'))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=10,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(bias_decay_mult=0.0,
+                                        norm_decay_mult=0.0,
+                                        custom_keys={
+                                            'backbone.text_model':
+                                            dict(lr_mult=0.01),
+                                            'logit_scale':
+                                            dict(weight_decay=0.0)
+                                        }),
+                     constructor='YOLOWv5OptimizerConstructor')

YOLO-World/configs/pretrain/yolo_world_v2_l_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py

@@ -0,0 +1,171 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_l_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'],
+                      allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 1203
+num_training_classes = 80
+max_epochs = 100  # Maximum training epochs
+close_mosaic_epochs = 2
+save_epoch_intervals = 2
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-3
+weight_decay = 0.05 / 2
+train_batch_size_per_gpu = 16
+# text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    dict(type='MultiModalMosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform,
+]
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *text_transform]
+obj365v1_train_dataset = dict(
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5Objects365V1Dataset',
+        data_root='data/objects365v1/',
+        ann_file='annotations/objects365_train.json',
+        data_prefix=dict(img='train/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/obj365v1_class_texts.json',
+    pipeline=train_pipeline)
+
+mg_train_dataset = dict(type='YOLOv5MixedGroundingDataset',
+                        data_root='data/mixed_grounding/',
+                        ann_file='annotations/final_mixed_train_no_coco.json',
+                        data_prefix=dict(img='gqa/images/'),
+                        filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                        pipeline=train_pipeline)
+
+flickr_train_dataset = dict(
+    type='YOLOv5MixedGroundingDataset',
+    data_root='data/flickr/',
+    ann_file='annotations/final_flickr_separateGT_train.json',
+    data_prefix=dict(img='full_images/'),
+    filter_cfg=dict(filter_empty_gt=True, min_size=32),
+    pipeline=train_pipeline)
+
+train_dataloader = dict(batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=dict(_delete_=True,
+                                     type='ConcatDataset',
+                                     datasets=[
+                                         obj365v1_train_dataset,
+                                         flickr_train_dataset, mg_train_dataset
+                                     ],
+                                     ignore_keys=['classes', 'palette']))
+
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5LVISV1Dataset',
+                 data_root='data/coco/',
+                 test_mode=True,
+                 ann_file='lvis/lvis_v1_minival_inserted_image_name.json',
+                 data_prefix=dict(img=''),
+                 batch_shapes_cfg=None),
+    class_text_path='data/texts/lvis_v1_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+
+val_evaluator = dict(type='mmdet.LVISMetric',
+                     ann_file='data/coco/lvis/lvis_v1_minival_inserted_image_name.json',
+                     metric='bbox')
+test_evaluator = val_evaluator
+
+# training settings
+default_hooks = dict(param_scheduler=dict(max_epochs=max_epochs),
+                     checkpoint=dict(interval=save_epoch_intervals,
+                                     rule='greater'))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=10,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(bias_decay_mult=0.0,
+                                        norm_decay_mult=0.0,
+                                        custom_keys={
+                                            'backbone.text_model':
+                                            dict(lr_mult=0.01),
+                                            'logit_scale':
+                                            dict(weight_decay=0.0)
+                                        }),
+                     constructor='YOLOWv5OptimizerConstructor')

YOLO-World/configs/pretrain/yolo_world_v2_l_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_val.py

@@ -0,0 +1,171 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_l_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'],
+                      allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 1203
+num_training_classes = 80
+max_epochs = 100  # Maximum training epochs
+close_mosaic_epochs = 2
+save_epoch_intervals = 2
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-3
+weight_decay = 0.05 / 2
+train_batch_size_per_gpu = 16
+# text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+text_model_name = 'openai/clip-vit-base-patch32'
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    dict(type='MultiModalMosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform,
+]
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *text_transform]
+obj365v1_train_dataset = dict(
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5Objects365V1Dataset',
+        data_root='data/objects365v1/',
+        ann_file='annotations/objects365_train.json',
+        data_prefix=dict(img='train/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/obj365v1_class_texts.json',
+    pipeline=train_pipeline)
+
+mg_train_dataset = dict(type='YOLOv5MixedGroundingDataset',
+                        data_root='data/mixed_grounding/',
+                        ann_file='annotations/final_mixed_train_no_coco.json',
+                        data_prefix=dict(img='gqa/images/'),
+                        filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                        pipeline=train_pipeline)
+
+flickr_train_dataset = dict(
+    type='YOLOv5MixedGroundingDataset',
+    data_root='data/flickr/',
+    ann_file='annotations/final_flickr_separateGT_train.json',
+    data_prefix=dict(img='full_images/'),
+    filter_cfg=dict(filter_empty_gt=True, min_size=32),
+    pipeline=train_pipeline)
+
+train_dataloader = dict(batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=dict(_delete_=True,
+                                     type='ConcatDataset',
+                                     datasets=[
+                                         obj365v1_train_dataset,
+                                         flickr_train_dataset, mg_train_dataset
+                                     ],
+                                     ignore_keys=['classes', 'palette']))
+
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5LVISV1Dataset',
+                 data_root='data/coco/',
+                 test_mode=True,
+                 ann_file='lvis/lvis_v1_val.json',
+                 data_prefix=dict(img=''),
+                 batch_shapes_cfg=None),
+    class_text_path='data/texts/lvis_v1_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+
+val_evaluator = dict(type='mmdet.LVISMetric',
+                     ann_file='data/coco/lvis/lvis_v1_val.json',
+                     metric='bbox')
+test_evaluator = val_evaluator
+
+# training settings
+default_hooks = dict(param_scheduler=dict(max_epochs=max_epochs),
+                     checkpoint=dict(interval=save_epoch_intervals,
+                                     rule='greater'))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=10,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(bias_decay_mult=0.0,
+                                        norm_decay_mult=0.0,
+                                        custom_keys={
+                                            'backbone.text_model':
+                                            dict(lr_mult=0.01),
+                                            'logit_scale':
+                                            dict(weight_decay=0.0)
+                                        }),
+                     constructor='YOLOWv5OptimizerConstructor')

YOLO-World/configs/pretrain/yolo_world_v2_m_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_1280ft_lvis_minival.py

@@ -0,0 +1,198 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_m_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'],
+                      allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 1203
+num_training_classes = 80
+max_epochs = 100  # Maximum training epochs
+close_mosaic_epochs = 2
+save_epoch_intervals = 2
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-3
+weight_decay = 0.05 / 2
+train_batch_size_per_gpu = 16
+text_model_name = '../pretrained_models/clip-vit-base-patch32-projection'
+img_scale = (1280, 1280)
+
+text_model_name = 'openai/clip-vit-base-patch32'
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    dict(type='MultiModalMosaic',
+         img_scale=img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border=(-img_scale[0] // 2, -img_scale[1] // 2),
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform,
+]
+
+train_pipeline_stage2 = [
+    *_base_.pre_transform,
+    dict(type='YOLOv5KeepRatioResize', scale=img_scale),
+    dict(
+        type='LetterResize',
+        scale=img_scale,
+        allow_scale_up=True,
+        pad_val=dict(img=114.0)),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform
+]
+obj365v1_train_dataset = dict(
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5Objects365V1Dataset',
+        data_root='data/objects365v1/',
+        ann_file='annotations/objects365_train.json',
+        data_prefix=dict(img='train/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/obj365v1_class_texts.json',
+    pipeline=train_pipeline)
+
+mg_train_dataset = dict(type='YOLOv5MixedGroundingDataset',
+                        data_root='data/mixed_grounding/',
+                        ann_file='annotations/final_mixed_train_no_coco.json',
+                        data_prefix=dict(img='gqa/images/'),
+                        filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                        pipeline=train_pipeline)
+
+flickr_train_dataset = dict(
+    type='YOLOv5MixedGroundingDataset',
+    data_root='data/flickr/',
+    ann_file='annotations/final_flickr_separateGT_train.json',
+    data_prefix=dict(img='full_images/'),
+    filter_cfg=dict(filter_empty_gt=True, min_size=32),
+    pipeline=train_pipeline)
+
+train_dataloader = dict(batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=dict(_delete_=True,
+                                     type='ConcatDataset',
+                                     datasets=[
+                                         obj365v1_train_dataset,
+                                         flickr_train_dataset, mg_train_dataset
+                                     ],
+                                     ignore_keys=['classes', 'palette']))
+
+test_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='YOLOv5KeepRatioResize', scale=img_scale),
+    dict(
+        type='LetterResize',
+        scale=img_scale,
+        allow_scale_up=False,
+        pad_val=dict(img=114)),
+    dict(type='LoadAnnotations', with_bbox=True, _scope_='mmdet'),
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5LVISV1Dataset',
+                 data_root='data/coco/',
+                 test_mode=True,
+                 ann_file='lvis/lvis_v1_minival_inserted_image_name.json',
+                 data_prefix=dict(img=''),
+                 batch_shapes_cfg=None),
+    class_text_path='data/texts/lvis_v1_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+
+val_evaluator = dict(type='mmdet.LVISMetric',
+                     ann_file='data/coco/lvis/lvis_v1_minival_inserted_image_name.json',
+                     metric='bbox')
+test_evaluator = val_evaluator
+
+# training settings
+default_hooks = dict(param_scheduler=dict(max_epochs=max_epochs),
+                     checkpoint=dict(interval=save_epoch_intervals,
+                                     rule='greater'))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=10,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(bias_decay_mult=0.0,
+                                        norm_decay_mult=0.0,
+                                        custom_keys={
+                                            'backbone.text_model':
+                                            dict(lr_mult=0.01),
+                                            'logit_scale':
+                                            dict(weight_decay=0.0)
+                                        }),
+                     constructor='YOLOWv5OptimizerConstructor')

YOLO-World/configs/pretrain/yolo_world_v2_m_vlpan_bn_2e-3_100e_4x8gpus_obj365v1_goldg_train_lvis_minival.py

@@ -0,0 +1,171 @@
+_base_ = ('../../third_party/mmyolo/configs/yolov8/'
+          'yolov8_m_syncbn_fast_8xb16-500e_coco.py')
+custom_imports = dict(imports=['yolo_world'],
+                      allow_failed_imports=False)
+
+# hyper-parameters
+num_classes = 1203
+num_training_classes = 80
+max_epochs = 100  # Maximum training epochs
+close_mosaic_epochs = 2
+save_epoch_intervals = 2
+text_channels = 512
+neck_embed_channels = [128, 256, _base_.last_stage_out_channels // 2]
+neck_num_heads = [4, 8, _base_.last_stage_out_channels // 2 // 32]
+base_lr = 2e-3
+weight_decay = 0.05 / 2
+train_batch_size_per_gpu = 16
+text_model_name = 'openai/clip-vit-large-patch14-336'
+text_model_name = 'openai/clip-vit-base-patch32'
+# model settings
+model = dict(
+    type='YOLOWorldDetector',
+    mm_neck=True,
+    num_train_classes=num_training_classes,
+    num_test_classes=num_classes,
+    data_preprocessor=dict(type='YOLOWDetDataPreprocessor'),
+    backbone=dict(
+        _delete_=True,
+        type='MultiModalYOLOBackbone',
+        image_model={{_base_.model.backbone}},
+        text_model=dict(
+            type='HuggingCLIPLanguageBackbone',
+            model_name=text_model_name,
+            frozen_modules=['all'])),
+    neck=dict(type='YOLOWorldPAFPN',
+              guide_channels=text_channels,
+              embed_channels=neck_embed_channels,
+              num_heads=neck_num_heads,
+              block_cfg=dict(type='MaxSigmoidCSPLayerWithTwoConv')),
+    bbox_head=dict(type='YOLOWorldHead',
+                   head_module=dict(type='YOLOWorldHeadModule',
+                                    use_bn_head=True,
+                                    embed_dims=text_channels,
+                                    num_classes=num_training_classes)),
+    train_cfg=dict(assigner=dict(num_classes=num_training_classes)))
+
+# dataset settings
+text_transform = [
+    dict(type='RandomLoadText',
+         num_neg_samples=(num_classes, num_classes),
+         max_num_samples=num_training_classes,
+         padding_to_max=True,
+         padding_value=''),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip',
+                    'flip_direction', 'texts'))
+]
+train_pipeline = [
+    *_base_.pre_transform,
+    dict(type='MultiModalMosaic',
+         img_scale=_base_.img_scale,
+         pad_val=114.0,
+         pre_transform=_base_.pre_transform),
+    dict(
+        type='YOLOv5RandomAffine',
+        max_rotate_degree=0.0,
+        max_shear_degree=0.0,
+        scaling_ratio_range=(1 - _base_.affine_scale, 1 + _base_.affine_scale),
+        max_aspect_ratio=_base_.max_aspect_ratio,
+        border=(-_base_.img_scale[0] // 2, -_base_.img_scale[1] // 2),
+        border_val=(114, 114, 114)),
+    *_base_.last_transform[:-1],
+    *text_transform,
+]
+train_pipeline_stage2 = [*_base_.train_pipeline_stage2[:-1], *text_transform]
+obj365v1_train_dataset = dict(
+    type='MultiModalDataset',
+    dataset=dict(
+        type='YOLOv5Objects365V1Dataset',
+        data_root='data/objects365v1/',
+        ann_file='annotations/objects365_train.json',
+        data_prefix=dict(img='train/'),
+        filter_cfg=dict(filter_empty_gt=False, min_size=32)),
+    class_text_path='data/texts/obj365v1_class_texts.json',
+    pipeline=train_pipeline)
+
+mg_train_dataset = dict(type='YOLOv5MixedGroundingDataset',
+                        data_root='data/mixed_grounding/',
+                        ann_file='annotations/final_mixed_train_no_coco.json',
+                        data_prefix=dict(img='gqa/images/'),
+                        filter_cfg=dict(filter_empty_gt=False, min_size=32),
+                        pipeline=train_pipeline)
+
+flickr_train_dataset = dict(
+    type='YOLOv5MixedGroundingDataset',
+    data_root='data/flickr/',
+    ann_file='annotations/final_flickr_separateGT_train.json',
+    data_prefix=dict(img='full_images/'),
+    filter_cfg=dict(filter_empty_gt=True, min_size=32),
+    pipeline=train_pipeline)
+
+train_dataloader = dict(batch_size=train_batch_size_per_gpu,
+                        collate_fn=dict(type='yolow_collate'),
+                        dataset=dict(_delete_=True,
+                                     type='ConcatDataset',
+                                     datasets=[
+                                         obj365v1_train_dataset,
+                                         flickr_train_dataset, mg_train_dataset
+                                     ],
+                                     ignore_keys=['classes', 'palette']))
+
+test_pipeline = [
+    *_base_.test_pipeline[:-1],
+    dict(type='LoadText'),
+    dict(type='mmdet.PackDetInputs',
+         meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                    'scale_factor', 'pad_param', 'texts'))
+]
+coco_val_dataset = dict(
+    _delete_=True,
+    type='MultiModalDataset',
+    dataset=dict(type='YOLOv5LVISV1Dataset',
+                 data_root='data/coco/',
+                 test_mode=True,
+                 ann_file='lvis/lvis_v1_minival_inserted_image_name.json',
+                 data_prefix=dict(img=''),
+                 batch_shapes_cfg=None),
+    class_text_path='data/texts/lvis_v1_class_texts.json',
+    pipeline=test_pipeline)
+val_dataloader = dict(dataset=coco_val_dataset)
+test_dataloader = val_dataloader
+
+val_evaluator = dict(type='mmdet.LVISMetric',
+                     ann_file='data/coco/lvis/lvis_v1_minival_inserted_image_name.json',
+                     metric='bbox')
+test_evaluator = val_evaluator
+
+# training settings
+default_hooks = dict(param_scheduler=dict(max_epochs=max_epochs),
+                     checkpoint=dict(interval=save_epoch_intervals,
+                                     rule='greater'))
+custom_hooks = [
+    dict(type='EMAHook',
+         ema_type='ExpMomentumEMA',
+         momentum=0.0001,
+         update_buffers=True,
+         strict_load=False,
+         priority=49),
+    dict(type='mmdet.PipelineSwitchHook',
+         switch_epoch=max_epochs - close_mosaic_epochs,
+         switch_pipeline=train_pipeline_stage2)
+]
+train_cfg = dict(max_epochs=max_epochs,
+                 val_interval=10,
+                 dynamic_intervals=[((max_epochs - close_mosaic_epochs),
+                                     _base_.val_interval_stage2)])
+optim_wrapper = dict(optimizer=dict(
+    _delete_=True,
+    type='AdamW',
+    lr=base_lr,
+    weight_decay=weight_decay,
+    batch_size_per_gpu=train_batch_size_per_gpu),
+                     paramwise_cfg=dict(bias_decay_mult=0.0,
+                                        norm_decay_mult=0.0,
+                                        custom_keys={
+                                            'backbone.text_model':
+                                            dict(lr_mult=0.01),
+                                            'logit_scale':
+                                            dict(weight_decay=0.0)
+                                        }),
+                     constructor='YOLOWv5OptimizerConstructor')