Initial clean SEMA Space package

.dockerignore

@@ -0,0 +1,15 @@
+.git
+.idea
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.pytest_cache
+.mypy_cache
+.venv
+venv
+dataset
+evaluation/results
+evaluation/eval_db
+output_keyframes
+webapp/.hf_space_build

.gitattributes

@@ -0,0 +1,3 @@
+*.db filter=lfs diff=lfs merge=lfs -text
+RAG/Knowledge_Database/languagebind_main/data/bpe_simple_vocab_16e6.txt.gz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.10-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV HF_HOME=/tmp/huggingface
+ENV TRANSFORMERS_CACHE=/tmp/huggingface
+ENV SEMA_TMP_DIR=/tmp
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ffmpeg \
+    libgl1 \
+    libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /app/requirements.txt
+COPY webapp/requirements.txt /app/webapp-requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt -r /app/webapp-requirements.txt
+
+COPY . /app
+
+EXPOSE 7860
+
+CMD ["uvicorn", "webapp.backend.app:app", "--host", "0.0.0.0", "--port", "7860"]

RAG/Knowledge_Database/AI_dbmanager.py

@@ -0,0 +1,750 @@
+"""
+@filename: AI_dbmanager.py
+@description: AI Database Operations Class
+"""
+
+import os
+import sys
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+if REPO_ROOT not in sys.path:
+    sys.path.insert(0, REPO_ROOT)
+
+# absolute_project_root = r"D:\Pythonworks\SpatialTemporalAttentionGCN-master\SpatialTemporalAttentionGCN-master"
+#
+# project_root = os.path.abspath(absolute_project_root)
+#
+# if project_root not in sys.path:
+#     sys.path.insert(0, project_root)
+
+from sqlalchemy import Column, Integer, String, Text, LargeBinary, Float, ForeignKey, MetaData, create_engine, inspect, text
+from sqlalchemy.orm import declarative_base, relationship, sessionmaker
+from openai import OpenAI
+try:
+    from deep_translator import DeeplTranslator
+except ImportError:
+    DeeplTranslator = None
+import pickle
+from RAG.Knowledge_Database.AIdbconfig import engine, session, db_pure_path
+import time
+import numpy as np
+import re
+
+rag_split_and_merge = None
+_rag_get_embedding_languagebind_video = None
+_rag_get_embedding_languagebind_text = None
+
+
+def _load_ragfunc_helpers():
+    global rag_split_and_merge
+    global _rag_get_embedding_languagebind_video
+    global _rag_get_embedding_languagebind_text
+
+    if (
+        rag_split_and_merge is not None
+        and _rag_get_embedding_languagebind_video is not None
+        and _rag_get_embedding_languagebind_text is not None
+    ):
+        return
+
+    from RAG.Knowledge_Database.RAGFunc import (
+        split_and_merge as imported_split_and_merge,
+        get_embedding_languagebind_video as imported_get_embedding_languagebind_video,
+        get_embedding_languagebind_text as imported_get_embedding_languagebind_text,
+    )
+
+    rag_split_and_merge = imported_split_and_merge
+    _rag_get_embedding_languagebind_video = imported_get_embedding_languagebind_video
+    _rag_get_embedding_languagebind_text = imported_get_embedding_languagebind_text
+
+
+def get_embedding(texts,model_name="text-embedding-v4"):
+    client = OpenAI(
+        api_key=os.getenv("ALI_API_KEY"),
+        base_url="https://dashscope.aliyuncs.com/compatible-mode/v1"
+    )
+    response = client.embeddings.create(
+        model=model_name,
+        input=texts,
+        dimensions=1024,
+        encoding_format="float"
+    )
+    return [np.array(item.embedding, dtype=np.float32) for item in response.data]
+
+
+def get_embedding_languagebind_video(video_path):
+    _load_ragfunc_helpers()
+    return _rag_get_embedding_languagebind_video(video_path)
+
+
+def get_embedding_languagebind_text(texts):
+    _load_ragfunc_helpers()
+    return _rag_get_embedding_languagebind_text(texts)
+
+Base = declarative_base()
+
+class Document(Base):
+    """Document table"""
+    __tablename__ = "documents"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    title = Column(String(200))
+    content = Column(Text)
+    title_embedding = Column(LargeBinary)
+
+    chunks = relationship("Chunk", back_populates="document")
+
+class Chunk(Base):
+    """Document chunk table"""
+    __tablename__ = "chunks"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    document_id = Column(Integer, ForeignKey("documents.id"))
+    text = Column(Text)
+
+    embedding = Column(LargeBinary)  # Store embedding vector (serialized into BLOB)
+
+    document = relationship("Document", back_populates="chunks")
+    
+class Embedding(Base):
+    """Embedding table for storing text vector representations"""
+    __tablename__ = "embeddings"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    chunk_id = Column(Integer, ForeignKey("chunks.id"))
+    vector = Column(LargeBinary)  # Store serialized vector
+    model_name = Column(String(100))  # Record the embedding model used
+    created_at = Column(Float)  # Record creation time
+
+    chunk = relationship("Chunk", back_populates="embeddings")
+
+Chunk.embeddings = relationship("Embedding", back_populates="chunk")
+
+SECTION_TITLE_RE = re.compile(r"(?m)^//\s*(.+?)\s*$")
+
+
+def ensure_database_schema(db_engine):
+    """Create missing tables and add title embedding column for legacy DBs."""
+    Base.metadata.create_all(db_engine)
+    inspector = inspect(db_engine)
+    table_names = set(inspector.get_table_names())
+    if "documents" not in table_names:
+        return
+
+    document_columns = {column["name"] for column in inspector.get_columns("documents")}
+    if "title_embedding" not in document_columns:
+        with db_engine.begin() as conn:
+            conn.execute(text("ALTER TABLE documents ADD COLUMN title_embedding BLOB"))
+
+
+def split_and_merge(text, chunk_size=500):
+    """Use the repo splitter when available; otherwise fall back to a lightweight local splitter."""
+    if rag_split_and_merge is None:
+        try:
+            _load_ragfunc_helpers()
+        except Exception:
+            pass
+
+    if rag_split_and_merge is not None:
+        return rag_split_and_merge(text, chunk_size=chunk_size)
+
+    normalized_text = str(text or "").replace("\r\n", "\n").strip()
+    if not normalized_text:
+        return []
+
+    paragraphs = [p.strip() for p in normalized_text.split("\n\n") if p.strip()]
+    chunks = []
+    sentence_re = re.compile(r"[^。！？；!?;\n]+[。！？；!?;]?")
+
+    for paragraph in paragraphs:
+        sentences = [s.strip() for s in sentence_re.findall(paragraph) if s.strip()]
+        if not sentences:
+            sentences = [paragraph]
+
+        current = ""
+        for sentence in sentences:
+            if len(current) + len(sentence) <= chunk_size:
+                current += sentence
+            else:
+                if current:
+                    chunks.append(current.strip())
+                current = sentence
+
+        if current:
+            chunks.append(current.strip())
+
+    return chunks
+
+
+def parse_structured_knowledge_sections(content):
+    """Parse `//title` knowledge blocks from zhishi-style text."""
+    text = str(content or "").replace("\r\n", "\n")
+    matches = list(SECTION_TITLE_RE.finditer(text))
+    if not matches:
+        return []
+
+    sections = []
+    for idx, match in enumerate(matches):
+        title = match.group(1).strip()
+        start = match.end()
+        end = matches[idx + 1].start() if idx + 1 < len(matches) else len(text)
+        body = text[start:end].strip()
+        if title and body:
+            sections.append((title, body))
+    return sections
+
+
+def build_section_chunks(title, body, chunk_size):
+    """Split one knowledge section into plain text chunks without duplicating the title."""
+    normalized_title = str(title or "").strip()
+    normalized_body = str(body or "").strip()
+    if not normalized_title or not normalized_body:
+        return []
+
+    section_chunks = split_and_merge(normalized_body, chunk_size=chunk_size)
+    if not section_chunks:
+        section_chunks = [normalized_body]
+
+    plain_chunks = []
+    for chunk_text in section_chunks:
+        normalized_chunk = str(chunk_text or "").strip()
+        if normalized_chunk:
+            plain_chunks.append(normalized_chunk)
+    return plain_chunks
+
+
+def build_document_chunks(title, content, chunk_size):
+    """Chunk structured KB text per `//title` block before sentence chunking."""
+    sections = parse_structured_knowledge_sections(content)
+    if not sections:
+        return build_section_chunks(title=title, body=content, chunk_size=chunk_size)
+
+    chunks = []
+    for section_title, section_body in sections:
+        chunks.extend(build_section_chunks(title=section_title, body=section_body, chunk_size=chunk_size))
+    return chunks
+
+
+def build_pure_knowledge_db(zhishi_path, db_path, chunk_size=260, rebuild=True):
+    """Build the pure knowledge DB from zhishi.txt using one document per knowledge block."""
+    zhishi_path = os.path.abspath(zhishi_path)
+    db_path = os.path.abspath(db_path)
+
+    if not os.path.exists(zhishi_path):
+        raise FileNotFoundError(f"Knowledge source file not found: {zhishi_path}")
+
+    os.makedirs(os.path.dirname(db_path), exist_ok=True)
+
+    pure_engine = create_engine(f"sqlite:///{db_path}", echo=False)
+    pure_session_factory = sessionmaker(bind=pure_engine)
+    ensure_database_schema(pure_engine)
+    pure_session = pure_session_factory()
+
+    if rebuild and os.path.exists(db_path):
+        try:
+            pure_session.close()
+            pure_engine.dispose()
+            os.remove(db_path)
+            pure_engine = create_engine(f"sqlite:///{db_path}", echo=False)
+            pure_session_factory = sessionmaker(bind=pure_engine)
+            ensure_database_schema(pure_engine)
+            pure_session = pure_session_factory()
+        except PermissionError:
+            pure_engine = create_engine(f"sqlite:///{db_path}", echo=False)
+            pure_session_factory = sessionmaker(bind=pure_engine)
+            pure_session = pure_session_factory()
+            ensure_database_schema(pure_engine)
+            pure_session.query(Embedding).delete()
+            pure_session.query(Chunk).delete()
+            pure_session.query(Document).delete()
+            pure_session.commit()
+
+    with open(zhishi_path, "r", encoding="utf-8") as f:
+        raw_text = f.read()
+
+    sections = parse_structured_knowledge_sections(raw_text)
+    if not sections:
+        raise ValueError(f"No structured `//title` sections found in {zhishi_path}")
+
+    document_count = 0
+    chunk_count = 0
+
+    try:
+        for section_title, section_body in sections:
+            document = Document(title=section_title, content=section_body)
+            pure_session.add(document)
+            pure_session.flush()
+
+            section_chunks = build_section_chunks(
+                title=section_title,
+                body=section_body,
+                chunk_size=chunk_size,
+            )
+            for chunk_text in section_chunks:
+                pure_session.add(
+                    Chunk(
+                        document_id=document.id,
+                        text=chunk_text,
+                    )
+                )
+
+            document_count += 1
+            chunk_count += len(section_chunks)
+
+        pure_session.commit()
+        return {
+            "db_path": db_path,
+            "zhishi_path": zhishi_path,
+            "sections": len(sections),
+            "documents": document_count,
+            "chunks": chunk_count,
+            "embeddings": 0,
+            "chunk_size": chunk_size,
+        }
+    except Exception:
+        pure_session.rollback()
+        raise
+    finally:
+        pure_session.close()
+        pure_engine.dispose()
+
+def batch_translate_files(input_folder, api_key, target_lang='en'):
+    """Batch translate text files using DeepL API"""
+    if DeeplTranslator is None:
+        raise ImportError("deep_translator is required for batch_translate_files but is not installed.")
+
+    translator = DeeplTranslator(api_key=api_key, source="zh", target=target_lang, use_free_api=True)
+    
+    output_folder = os.path.join(input_folder, "translated_results")
+    if not os.path.exists(output_folder):
+        os.makedirs(output_folder)
+
+    for filename in os.listdir(input_folder):
+        if filename.endswith(".txt"):
+            file_path = os.path.join(input_folder, filename)
+            
+            print(f"Processing: {filename}...")
+            
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    content = f.read().replace('\n', ' ').strip()
+                
+                if not content:
+                    continue
+
+                translated_text = translator.translate(content)
+                
+                output_path = os.path.join(output_folder, f"translated_{filename}")
+                with open(output_path, 'w', encoding='utf-8') as f:
+                    f.write(translated_text)
+                
+                print(f"Successfully exported: translated_{filename}")
+
+            except Exception as e:
+                print(f"Failed to process {filename}: {str(e)}")
+
+class KnowledgeDB:
+    def __init__(self, session):
+        self.engine = session.bind if getattr(session, "bind", None) is not None else engine
+        self.session = session
+        self._ensure_database_schema()
+
+    def _ensure_database_schema(self):
+        """Create missing tables and add title embedding column for legacy DBs."""
+        ensure_database_schema(self.engine)
+
+    @staticmethod
+    def _vector_to_bytes(vector):
+        return pickle.dumps(np.asarray(vector, dtype=np.float32))
+
+    @staticmethod
+    def _vector_from_bytes(blob):
+        return np.asarray(pickle.loads(blob), dtype=np.float32).flatten()
+
+    @staticmethod
+    def _cosine_similarity(vec, query_vec):
+        vec = np.asarray(vec, dtype=np.float32).flatten()
+        query_vec = np.asarray(query_vec, dtype=np.float32).flatten()
+        denom = np.linalg.norm(vec) * np.linalg.norm(query_vec)
+        if denom <= 0:
+            return -1.0
+        return float(np.dot(vec, query_vec) / denom)
+
+    def _get_query_vector(self, query, embed_fn):
+        query_text = str(query or "").strip()
+        if not query_text:
+            raise ValueError("query must not be empty")
+        return np.asarray(embed_fn([query_text])[0], dtype=np.float32).flatten()
+
+    def _get_chunk_embeddings_map(self, model_name="default", document_ids=None):
+        query = self.session.query(Chunk, Embedding).join(
+            Embedding, Embedding.chunk_id == Chunk.id
+        ).filter(Embedding.model_name == model_name)
+
+        if document_ids is not None:
+            doc_ids = [int(doc_id) for doc_id in document_ids]
+            if not doc_ids:
+                return {}
+            query = query.filter(Chunk.document_id.in_(doc_ids))
+
+        embeddings_map = {}
+        for chunk, embedding in query.all():
+            existing = embeddings_map.get(chunk.id)
+            if existing is None or embedding.created_at >= existing[1].created_at:
+                embeddings_map[chunk.id] = (chunk, embedding)
+        return embeddings_map
+        
+    def add_document(self, title, content, embed_fn=None, model_name="default", chunk_size=260):
+        """Add document and split into chunks"""
+        doc = Document(title=title, content=content)
+        self.session.add(doc)
+        self.session.commit()
+
+        chunks = build_document_chunks(title=title, content=content, chunk_size=chunk_size)
+
+        for text in chunks:
+            chunk = Chunk(
+                document_id=doc.id,
+                text=text
+            )
+            self.session.add(chunk)
+
+        self.session.commit()
+        return doc
+
+    def clear_chunks(self):
+        """Clear all chunks and related embeddings"""
+        try:
+            self.session.query(Embedding).delete()
+            self.session.query(Chunk).delete()
+            self.session.commit()
+            print("All chunks and embeddings cleared")
+        except Exception as e:
+            self.session.rollback()
+            print(f"Error clearing chunks: {e}")
+
+    def scan_and_add_chunks(self, chunk_size=260):
+        """Scan all documents and regenerate chunks"""
+        try:
+            documents = self.session.query(Document).all()
+            print(f"Found {len(documents)} documents, generating chunks...")
+            
+            count = 0
+            for doc in documents:
+                if not doc.content:
+                    continue
+                    
+                chunks = build_document_chunks(title=doc.title, content=doc.content, chunk_size=chunk_size)
+                for text in chunks:
+                    chunk = Chunk(
+                        document_id=doc.id,
+                        text=text
+                    )
+                    self.session.add(chunk)
+                    count += 1
+            
+            self.session.commit()
+            print(f"Generated {count} chunks for {len(documents)} documents")
+        except Exception as e:
+            self.session.rollback()
+            print(f"Error generating chunks: {e}")
+    
+    def delete_document(self, document_id: int):
+        """Delete document and its related chunks and embeddings"""
+        doc = self.session.query(Document).filter(Document.id == document_id).first()
+        if not doc:
+            print(f"Document {document_id} does not exist")
+            return False
+            
+        chunks = self.session.query(Chunk).filter(Chunk.document_id == document_id).all()
+        for chunk in chunks:
+            self.session.query(Embedding).filter(Embedding.chunk_id == chunk.id).delete()
+            
+        self.session.query(Chunk).filter(Chunk.document_id == document_id).delete()
+        self.session.delete(doc)
+        self.session.commit()
+        return True
+    
+    def delete_embedding(self):
+        """Delete all embeddings"""
+        embeddings = self.session.query(Embedding).all()
+        for embedding in embeddings:
+            self.session.delete(embedding)
+        self.session.query(Chunk).update({Chunk.embedding: None}, synchronize_session=False)
+        self.session.commit()
+        return True
+
+    def sync_chunk_embedding_column_from_embeddings(self, model_name="default"):
+        """
+        Temporary helper: mirror the latest vector in `embeddings` back into `chunks.embedding`.
+        Because `chunks.embedding` is a single column, it only stores one snapshot per chunk.
+        """
+        print(f"Syncing chunk.embedding from embeddings (model={model_name})...")
+        embeddings_map = self._get_chunk_embeddings_map(model_name=model_name)
+        if not embeddings_map:
+            print("No matching embeddings found to sync.")
+            return 0
+
+        updated_count = 0
+        try:
+            for chunk, embedding in embeddings_map.values():
+                chunk.embedding = embedding.vector
+                updated_count += 1
+            self.session.commit()
+            print(f"Synchronized {updated_count} chunk embeddings.")
+            return updated_count
+        except Exception:
+            self.session.rollback()
+            raise
+
+    def update_embeddings(self, embed_fn, model_name="default", batch_size=10):
+        """Update title embeddings for documents and text embeddings for chunks."""
+        if embed_fn is None:
+            raise ValueError("embed_fn must not be None")
+        batch_size = max(1, min(int(batch_size), 10))
+
+        print("Starting to update title and chunk embeddings...")
+        documents = self.session.query(Document).all()
+        chunks = self.session.query(Chunk).all()
+        successful_title_batches = 0
+        successful_chunk_batches = 0
+
+        if not documents and not chunks:
+            print("No documents or chunks found to update.")
+            return
+
+        self.session.query(Embedding).filter(Embedding.model_name == model_name).delete()
+        self.session.query(Document).update({Document.title_embedding: None}, synchronize_session=False)
+        self.session.query(Chunk).update({Chunk.embedding: None}, synchronize_session=False)
+        self.session.commit()
+
+        if documents:
+            for i in range(0, len(documents), batch_size):
+                batch_documents = documents[i:i + batch_size]
+                titles_to_embed = [str(doc.title or "").strip() for doc in batch_documents]
+                print(f"Processing title batch {i // batch_size + 1}: Sending {len(titles_to_embed)} titles to the embedding model...")
+                try:
+                    title_vectors = embed_fn(titles_to_embed)
+                    if len(title_vectors) != len(batch_documents):
+                        raise ValueError(
+                            f"Mismatch in count for title batch: Received {len(title_vectors)} embeddings for {len(batch_documents)} documents."
+                        )
+                    for document, vector in zip(batch_documents, title_vectors):
+                        document.title_embedding = self._vector_to_bytes(vector)
+                    self.session.commit()
+                    successful_title_batches += 1
+                except Exception as e:
+                    print(f"An error occurred during title batch {i // batch_size + 1}: {e}")
+                    self.session.rollback()
+                    continue
+
+        if not chunks:
+            print("No chunks found to update after title embedding generation.")
+            return
+
+        for i in range(0, len(chunks), batch_size):
+            batch_chunks = chunks[i:i + batch_size]
+            texts_to_embed = [chunk.text for chunk in batch_chunks]
+
+            print(f"Processing chunk batch {i // batch_size + 1}: Sending {len(texts_to_embed)} text chunks to the embedding model...")
+            try:
+                embedding_vectors = embed_fn(texts_to_embed)
+
+                if len(embedding_vectors) != len(batch_chunks):
+                    raise ValueError(
+                        f"Mismatch in count for batch: Received {len(embedding_vectors)} embeddings for {len(batch_chunks)} chunks."
+                    )
+
+                for chunk, vector in zip(batch_chunks, embedding_vectors):
+                    vector_blob = self._vector_to_bytes(vector)
+                    chunk.embedding = vector_blob
+                    embedding = Embedding(
+                        chunk=chunk,
+                        vector=vector_blob,
+                        model_name=model_name,
+                        created_at=time.time()
+                    )
+                    self.session.add(embedding)
+                self.session.commit()
+                successful_chunk_batches += 1
+
+            except Exception as e:
+                print(f"An error occurred during chunk batch {i // batch_size + 1}: {e}")
+                self.session.rollback()
+                continue
+
+        if documents and successful_title_batches == 0:
+            raise RuntimeError("Failed to generate any document title embeddings.")
+        if chunks and successful_chunk_batches == 0:
+            raise RuntimeError("Failed to generate any chunk embeddings.")
+
+        print("All embeddings updated successfully.")
+
+    def clear_database(self):
+        """Clear all data in database tables without deleting table structure"""
+        meta = MetaData()
+        meta.reflect(bind=self.engine)
+        with self.session.begin():
+            for table in reversed(meta.sorted_tables):
+                self.session.execute(table.delete())
+        self.session.commit()
+        print("Database cleared")
+        
+    def search(self, query, embed_fn, model_name="default", top_k=3):
+        """Search for similar content"""
+        query_vec = self._get_query_vector(query, embed_fn)
+        results = []
+
+        for chunk, embedding in self._get_chunk_embeddings_map(model_name=model_name).values():
+            score = self._cosine_similarity(self._vector_from_bytes(embedding.vector), query_vec)
+            if score > 0.5:
+                results.append((chunk, score))
+                    
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results[:top_k]
+
+    def search_document_titles(self, query, embed_fn, title_top_k=3):
+        """Search similar document titles using cached title embeddings."""
+        query_vec = self._get_query_vector(query, embed_fn)
+        results = []
+        documents = self.session.query(Document).filter(Document.title_embedding.isnot(None)).all()
+
+        for document in documents:
+            score = self._cosine_similarity(self._vector_from_bytes(document.title_embedding), query_vec)
+            results.append((document, score))
+
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results[:title_top_k]
+
+    def search_chunks_within_documents(self, query, document_ids, embed_fn, model_name="default", chunk_top_k=5):
+        """Search chunks only within the given document ids."""
+        query_vec = self._get_query_vector(query, embed_fn)
+        results = []
+
+        for chunk, embedding in self._get_chunk_embeddings_map(model_name=model_name, document_ids=document_ids).values():
+            score = self._cosine_similarity(self._vector_from_bytes(embedding.vector), query_vec)
+            results.append((chunk, score))
+
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results[:chunk_top_k]
+
+    def search_knowledge_two_stage(self, query, embed_fn, model_name="default", title_top_k=5, chunk_top_k=8):
+        """Retrieve top titles first, then search chunks inside those title-selected documents."""
+        title_results = self.search_document_titles(query, embed_fn=embed_fn, title_top_k=title_top_k)
+        document_ids = [document.id for document, _ in title_results]
+        
+        if not document_ids:
+            return self.search(query, embed_fn=embed_fn, model_name=model_name, top_k=chunk_top_k)
+
+        chunk_results = self.search_chunks_within_documents(
+            query=query,
+            document_ids=document_ids,
+            embed_fn=embed_fn,
+            model_name=model_name,
+            chunk_top_k=chunk_top_k,
+        )
+        if chunk_results:
+            return chunk_results
+        return self.search(query, embed_fn=embed_fn, model_name=model_name, top_k=chunk_top_k)
+
+    def from_video_search(self, query_vec, model_name="default", top_k=3):
+        """Search for similar content from video embedding"""
+        results = []
+        
+        chunks = self.session.query(Chunk).join(Embedding).filter(
+            Embedding.model_name == model_name
+        ).all()
+        
+        for chunk in chunks:
+            embedding = self.session.query(Embedding).filter(
+                Embedding.chunk_id == chunk.id,
+                Embedding.model_name == model_name
+            ).order_by(Embedding.created_at.desc()).first()
+            
+            if embedding:
+                vec = pickle.loads(embedding.vector)
+                vec = vec.flatten()
+                query_vec = query_vec.flatten()
+                score = np.dot(vec, query_vec) / (np.linalg.norm(vec) * np.linalg.norm(query_vec))
+                # print(f"score:{score}")
+                if score > 0.1:
+                    results.append(chunk.text)
+                    
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results[:top_k]
+    
+    def embedding_model_choose(self, model):
+        """Choose embedding model"""
+        if model == "ali-text-embedding-v3":
+            return get_embedding
+        elif model == 'languagebind_video':
+            return get_embedding_languagebind_video
+        elif model == 'languagebind_text':
+            return get_embedding_languagebind_text
+        
+    
+    def chat(self, user_question: str, top_k=3, embedding_model="ali-text-embedding-v3", chat_model="deepseek"):
+        """
+        Handle chat logic: search context, call LLM, and print response
+        """
+        print(f"Received question: {user_question}")
+        print("Retrieving relevant information...")
+
+        try:
+            search_results = self.search(
+                user_question,
+                embed_fn=self.embedding_model_choose(embedding_model),
+                model_name=embedding_model,
+                top_k=top_k
+            )
+        except Exception as e:
+            print(f"Error during information retrieval: {e}")
+            return
+
+        if not search_results:
+            print("No relevant information found.")
+            return
+
+        context_chunks = [chunk.text for chunk, score in search_results]
+        context = "\n\n---\n\n".join(context_chunks)
+
+        print("Context found, generating response...")
+        
+        try:
+            response = self.chat_model_choose(chat_model)(prompt=user_question, context=context)
+        except Exception as e:
+            print(f"Error calling LLM: {e}")
+            return
+
+        print("\nModel response:")
+        print(response)
+        
+# if __name__ == "__main__":
+#     rag_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+#     zhishi_path = os.path.join(rag_root, "zhishi.txt")
+#     stats = build_pure_knowledge_db(
+#         zhishi_path=zhishi_path,
+#         db_path=db_pure_path,
+#         chunk_size=200,
+#         rebuild=True,
+#     )
+
+#     print("Pure knowledge DB rebuild complete.")
+#     print(f"Source: {stats['zhishi_path']}")
+#     print(f"Target DB: {stats['db_path']}")
+#     print(f"Sections: {stats['sections']}")
+#     print(f"Documents: {stats['documents']}")
+#     print(f"Chunks: {stats['chunks']}")
+#     print(f"Embeddings: {stats['embeddings']}")
+# if __name__ == "__main__":
+    # from RAG.Knowledge_Database.AIdbconfig import session_pure
+
+    # db = KnowledgeDB(session_pure)
+    # # Temporary one-off backfill for existing rows, if needed:
+    # # db.sync_chunk_embedding_column_from_embeddings(model_name="ali-text-embedding-v4")
+    # db.update_embeddings(
+    #     embed_fn=get_embedding,
+    #     model_name="ali-text-embedding-v4",
+    #     batch_size=10,
+    # )
+

RAG/Knowledge_Database/AIdbconfig.py

@@ -0,0 +1,32 @@
+"""
+@filename: dbconfig.py
+@description: AI database file interface for loading AI database
+"""
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+# Get absolute path of current script file (AIdbconfig.py)
+script_path = os.path.abspath(__file__)
+# Get directory containing the script
+script_dir = os.path.dirname(script_path)
+# Get project root directory (parent of script directory)
+project_root = os.path.dirname(script_dir)
+# Construct absolute path to database file
+db_path = os.path.join(project_root, "db_files", "LLM_Knowledge_New_ali_zh.db")
+db_en_path = os.path.join(project_root, "db_files", "LLM_Knowledge_New_En.db")
+db_pure_path = os.path.join(project_root, "db_files", "LLM_Knowledge_Pure.db")
+
+DATABASE_URL = 'sqlite:///' + db_path
+DATABASE_EN_URL = 'sqlite:///' + db_en_path
+DATABASE_PURE_URL = 'sqlite:///' + db_pure_path
+
+engine = create_engine(DATABASE_URL, echo=False)
+engine_en = create_engine(DATABASE_EN_URL, echo=False)
+engine_pure = create_engine(DATABASE_PURE_URL, echo=False)
+
+# Create Session class
+session = sessionmaker(bind=engine)()
+session_en = sessionmaker(bind=engine_en)()
+session_pure = sessionmaker(bind=engine_pure)()

RAG/Knowledge_Database/RAGFunc.py

@@ -0,0 +1,1511 @@
+"""
+@filename: RAG.py
+@description: Implementation of all RAG functions
+"""
+import os
+import sys
+import re
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+import numpy as np
+from openai import OpenAI
+import requests
+import json
+from RAG.Knowledge_Database.languagebind_main.languagebind import LanguageBindImageTokenizer,LanguageBind, to_device, LanguageBindVideo, LanguageBindVideoTokenizer, LanguageBindVideoProcessor, transform_dict
+import torch
+import json
+from typing import List, Union, Dict, Any, Literal
+import base64
+from RTMPose.Bone_Feature_Extract import *
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+def split_and_merge(text, chunk_size=500):
+    """
+    Split text using LangChain, then merge into chunks close to chunk_size
+    - Does not cross paragraphs
+    - Does not break sentences
+    - Punctuation at end goes to previous chunk
+    - Remove leading punctuation from chunks
+    """
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=0,
+        separators=["\n\n", "\n", ".", ".", "！", "!", "？", "?", ";", ";"]
+    )
+    pieces = splitter.split_text(text)
+
+    chunks = []
+    current_chunk = ""
+    current_len = 0
+    punctuations = {".", ".", "！", "!", "？", "?", ";", ";"}
+
+    for piece in pieces:
+        piece = piece.strip()
+        if not piece:
+            continue
+
+        if "\n\n" in piece:
+            if current_chunk:
+                chunks.append(current_chunk.strip())
+                current_chunk = ""
+                current_len = 0
+            piece = piece.replace("\n\n", "").strip()
+            if not piece:
+                continue
+
+        if piece in punctuations:
+            current_chunk += piece
+            current_len += len(piece)
+            continue
+
+        piece_len = len(piece)
+        if current_len + piece_len <= chunk_size:
+            current_chunk += piece
+            current_len += piece_len
+        else:
+            if current_chunk:
+                while current_chunk and current_chunk[0] in punctuations:
+                    current_chunk = current_chunk[1:].lstrip()
+                if current_chunk:
+                    chunks.append(current_chunk.strip())
+            current_chunk = piece
+            current_len = piece_len
+
+    if current_chunk:
+        while current_chunk and current_chunk[0] in punctuations:
+            current_chunk = current_chunk[1:].lstrip()
+        if current_chunk:
+            chunks.append(current_chunk.strip())
+
+    return chunks
+
+def get_embedding(texts):
+    """Call OpenAI API to get text embedding vectors"""
+    client = OpenAI(
+        api_key=os.getenv("ALI_API_KEY"),
+        base_url="https://dashscope.aliyuncs.com/compatible-mode/v1"
+    )
+    response = client.embeddings.create(
+        model="text-embedding-v4",
+        input=texts,
+        dimensions=1024,
+        encoding_format="float"
+    )
+    embeddings = [np.array(item.embedding, dtype=np.float32) for item in response.data]
+    return embeddings
+
+
+def video_to_base64(video_path):
+    with open(video_path, "rb") as video_file:
+        encoded = base64.b64encode(video_file.read()).decode("utf-8")
+    return encoded
+
+def _media_type_from_ext(path: str) -> str:
+    ext = os.path.splitext(str(path))[1].lower()
+    if ext in (".jpg", ".jpeg"):
+        return "image/jpeg"
+    if ext == ".png":
+        return "image/png"
+    return "image/jpeg"
+
+
+def _load_openai_images_from_folder(folder: str) -> list[dict]:
+    folder_path = os.path.abspath(str(folder or "").strip())
+    if not folder_path or not os.path.isdir(folder_path):
+        raise ValueError(
+            f"Template keyframes directory not found: {folder_path}. "
+            "Please run extract_reference_keyframes.py first."
+        )
+
+    ordered_paths = []
+    numeric_files = []
+    for name in os.listdir(folder_path):
+        full_path = os.path.join(folder_path, name)
+        stem, ext = os.path.splitext(name)
+        if not os.path.isfile(full_path):
+            continue
+        # Pipeline 4 template keyframes: only *.jpg named as integers (1.jpg, 2.jpg, ...).
+        if ext.lower() != ".jpg":
+            continue
+        stem_stripped = str(stem).strip()
+        if not stem_stripped.isdigit():
+            continue
+        numeric_files.append((int(stem_stripped), os.path.abspath(full_path)))
+
+    numeric_files.sort(key=lambda x: x[0])
+    ordered_paths = [p for _, p in numeric_files]
+
+    if not ordered_paths:
+        raise ValueError(
+            f"No numeric .jpg template keyframes found in {folder_path}. "
+            "Expected files like 1.jpg, 2.jpg, 3.jpg."
+        )
+
+    openai_images = []
+    for img_path in ordered_paths:
+        try:
+            with open(img_path, "rb") as f:
+                b64 = base64.b64encode(f.read()).decode("utf-8")
+        except Exception as e:
+            raise ValueError(f"Failed to read template keyframe image: {img_path}. {e}") from e
+
+        media_type = _media_type_from_ext(img_path)
+        openai_images.append(
+            {
+                "type": "input_image",
+                "image_url": f"data:{media_type};base64,{b64}",
+            }
+        )
+
+    return openai_images
+
+
+def get_video_ori_keywords(
+    video_path,
+    pipeline=1,
+    target_k=5,
+    model_name='qwen3-vl-plus',
+    language='zh',
+    show=False,
+    template_keyframes_dir = None,
+) -> dict:
+    """
+    Multi-stage multimodal assessment for archery posture.
+
+    Pipeline semantics:
+        1) Video only: user content contains ONLY video_url (no extra text).
+        2) Keyframes only: extracted keyframes are sent as input_image parts (Responses API).
+        3) Keyframes + biomechanics metrics: keyframes + metrics text (Responses API).
+        4) Keyframes + biomechanics metrics + template keyframes loaded from folder.
+
+    Return:
+        A normalized assessment dict with keys:
+            total_score, head_score, hand_score, torso_score, foot_score, arm_score, comment
+        where `comment` is a keyword list split by comma.
+    """
+    def _normalize_language(lang_in: str) -> str:
+        lang_norm = str(lang_in or "en").strip().lower()
+        if lang_norm in ("zh", "zh-cn", "zh_hans", "cn", "chinese", "中文"):
+            return "zh"
+        if lang_norm in ("en", "en-us", "english"):
+            return "en"
+        raise ValueError("language must be 'en' or 'zh'")
+
+    def _chat_completion_output_text(resp) -> str:
+        if resp is None or not getattr(resp, "choices", None):
+            raise ValueError("Chat completion returned empty response.")
+
+        message = getattr(resp.choices[0], "message", None)
+        content = getattr(message, "content", None)
+        if isinstance(content, str) and content.strip():
+            return content.strip()
+
+        if isinstance(content, list):
+            parts = []
+            for item in content:
+                if isinstance(item, dict):
+                    if item.get("type") == "text" and item.get("text"):
+                        parts.append(str(item["text"]))
+                else:
+                    text = getattr(item, "text", None)
+                    if text:
+                        parts.append(str(text))
+            if parts:
+                return "".join(parts).strip()
+
+        raise ValueError("Chat completion contains no textual content.")
+
+    def _to_chat_content(input_content: list[dict]) -> list[dict]:
+        chat_content = []
+        for item in input_content:
+            if not isinstance(item, dict):
+                continue
+            item_type = item.get("type")
+            if item_type == "input_text":
+                chat_content.append({"type": "text", "text": str(item.get("text", ""))})
+            elif item_type == "input_image":
+                image_url = item.get("image_url")
+                if isinstance(image_url, dict):
+                    image_url = image_url.get("url")
+                if image_url:
+                    chat_content.append({"type": "image_url", "image_url": {"url": str(image_url)}})
+        if not chat_content:
+            raise ValueError("No valid chat content converted from keyframe inputs.")
+        return chat_content
+
+    def _extract_json_object_text(raw_text: str) -> str:
+        text = str(raw_text or "").strip()
+        if not text:
+            raise ValueError("Model response is empty; expected a JSON object string.")
+
+        decoder = json.JSONDecoder()
+        for idx, ch in enumerate(text):
+            if ch != "{":
+                continue
+            try:
+                parsed, end = decoder.raw_decode(text[idx:])
+            except json.JSONDecodeError:
+                continue
+            if isinstance(parsed, dict):
+                return text[idx: idx + end]
+        raise ValueError("No valid JSON object found in model response.")
+
+    def _normalize_comment_keywords(raw_comment) -> list[str]:
+        if isinstance(raw_comment, list):
+            keywords = [str(item).strip() for item in raw_comment]
+            return [k for k in keywords if k]
+
+        text = str(raw_comment or "").replace(",", ",")
+        parts = [p.strip() for p in text.split(",")]
+        return [p for p in parts if p]
+
+    def _parse_assessment_payload(raw_text: str) -> dict:
+        required_keys = (
+            "total_score",
+            "head_score",
+            "hand_score",
+            "torso_score",
+            "foot_score",
+            "arm_score",
+            "comment",
+        )
+        json_text = _extract_json_object_text(raw_text)
+        try:
+            payload = json.loads(json_text)
+        except Exception as e:
+            raise ValueError(f"Failed to parse model JSON response: {e}") from e
+
+        if not isinstance(payload, dict):
+            raise ValueError("Model response JSON is not an object.")
+
+        missing = [k for k in required_keys if k not in payload]
+        if missing:
+            raise ValueError(f"Model response JSON missing required keys: {missing}")
+
+        out = {}
+        for key in required_keys[:-1]:
+            try:
+                out[key] = int(payload[key])
+            except Exception as e:
+                raise ValueError(f"Invalid score field '{key}': {payload.get(key)}") from e
+        out["comment"] = _normalize_comment_keywords(payload.get("comment"))
+        return out
+
+    lang = _normalize_language(language)
+    pipeline = int(pipeline)
+    if pipeline not in (1, 2, 3, 4):
+        raise ValueError(
+            "pipeline must be 1 (video), 2 (keyframes), 3 (keyframes+metrics), "
+            "or 4 (keyframes+metrics+template keyframes)"
+        )
+
+
+    # client = OpenAI(
+    #         base_url="https://openrouter.ai/api/v1",
+    #         api_key=os.environ.get("OPENROUTER_API_KEY"),
+    #     )
+    client = OpenAI(
+        api_key=os.getenv("ALI_API_KEY"),
+        base_url="https://dashscope.aliyuncs.com/compatible-mode/v1"
+    )
+
+    system_base_map = {
+        "en": (
+            "You are a professional archery coach.\n"
+            "Use ONLY the user-provided inputs (video/keyframes and optional biomechanics metrics).\n"
+            "You must NOT search for information or draw upon your own prior knowledge.\n"
+            "Output MUST be ONLY a JSON object string with keys: "
+            "'total_score', 'head_score', 'hand_score', 'torso_score', 'foot_score', 'arm_score', 'comment'.\n"
+            "All scores MUST be integers: each part score in [0,5], total_score in [0,25].\n"
+            "total_score MUST equal head_score + hand_score + torso_score + foot_score + arm_score.\n"
+            "In 'comment', output a concise keyword sequence.\n"
+            "Use short phrases separated by commas.\n"
+            "Do not add extra keys, markdown, code fences, or any other text."
+        ),
+        "zh": (
+            "你是一名专业的射箭教练.\n"
+            "请参考下文完成指定的任务,下面先提出对输入及输出的要求.\n"
+            "只能使用用户提供的输入(视频/关键帧以及可选的人体生物力学指标),不得搜索或引用已知知识,也不能输出不基于实时评判的结果\n"
+            "输出必须且只能是一个 JSON 对象字符串,键必须为:"
+            "'total_score', 'head_score', 'hand_score', 'torso_score', 'foot_score', 'arm_score', 'comment'.\n"
+            "所有分数必须为整数:各部位分数范围 [0,5],总分 total_score 范围 [0,25].\n"
+            "comment字段输出简洁的关键句序列\n"
+            "用短语,使用&号分隔各序列中的关键句(关键句1&关键句2......这样的形式).\n"
+            "不要添加额外键、Markdown、代码块或任何其他文本"
+        ),
+    }
+
+    system_task_video_map = {
+        "en": (
+            "\nTask:\n"
+            "You will receive a full archery video. Observe and judge whether the archer achieves the following.\n"
+            "Static / aiming-phase observations (mainly from earlier parts):\n"
+            "- Body upright/vertical axis stable\n"
+            "- Head rotates sufficiently toward target\n"
+            "- Arms form a straight line / alignment is correct\n"
+            "- Rear hand firmly anchored against the chin\n"
+            "- Nose touches the bowstring (string touches chin/nose)\n"
+            "- Bow hand fingers are relaxed (not clenched)\n"
+            "- Before release, rear hand does NOT slip/loosen the string during aiming\n"
+            "Release / follow-through observations (around release and after):\n"
+            "- At release, rear hand extends backward (follow-through)\n"
+            "- At release, front/bow hand relaxes and pushes/lets the bow move naturally (not grabbing)\n"
+            "Then output the required JSON with preliminary scores and a keyword-style comment.\n"
+            "Keywords MUST be concise, and MUST be selected from the following keyword library (use whichever items apply).\n"
+            "Keyword library: separate keywords with \",\".\n"
+            "1. Torso: (1) Correct: Body vertical axis is perpendicular to the ground; body is side-on to the shooting direction; "
+            "(2) Incorrect: Body center of mass is tilted/leaning; body is not rotated to a side-on stance.\n"
+            "2. Arms: (1) Correct: Front hand, front shoulder, and rear shoulder are aligned on one line; front forearm is internally rotated; "
+            "rear forearm is aligned with the arrow; "
+            "(2) Incorrect: Front shoulder has an angle and is not opened; forearm is turned outward; forearm is not aligned with the arrow.\n"
+            "3. Head: (1) Correct: Head rotates horizontally; (2) Incorrect: Head does not rotate enough.\n"
+            "4. Feet: (1) Correct: Feet stance is shoulder-width; (2) Incorrect: Feet are placed arbitrarily.\n"
+            "5. Hands: (1) Correct: Bow-hand fingers are relaxed; rear hand is anchored at the cheek/jaw area; "
+            "the bowstring touches the chin and nose; during aiming the rear hand does not slip/loosen the string; "
+            "at release the rear hand extends backward; at release the front/bow hand relaxes and pushes the bow; "
+            "(2) Incorrect: Fingers are stiff or clenched; rear hand is floating and not anchored; string does not touch chin and nose; "
+            "before/after release the rear hand slowly slides backward; at release the rear hand stays in place or moves outward/forward; "
+            "at release the front/bow hand grabs the bow.\n"
+        ),
+        "zh": (
+            "\n任务:\n"
+            "你将看到一个射箭短视频\n"
+            "从视频中观察并判断下面的动作,如果符合则comment中严格加入(一字不差)对应标准后第一个括号内关键句库中的正确动作(R),反之加上对应的错误动作(W),并按照评分规则(S)对分数进行初评,再按下文中的信息修正你的评价(如果出现)\n"
+            "1.身体是否直立(R:身体纵轴垂直于地面/W:身体重心歪斜)(S:可认为基本垂直的情况躯干部位给4/5分,出现歪斜给2/3分)\n"
+            "2.身体是否转到侧对(S:一般给出R:身体侧向对着射箭方向 的评价,除非极其离谱,给出W:身体没有转到侧对)\n"
+            "3.头部是否转动到位(R:头部水平转动/W:头部没有转动到位)(S:转动到位给5分,不到位给3分,极其离谱情况给1/2分)\n"
+            "4.手臂是否成一条直线(R:前手,前肩,后肩在一条线上,后手小臂与箭在一条线上/W:小臂与箭有角度)(S:后肩部有角度给2/3分,否则4/5分)\n"
+            "5.后手是否牢牢靠在下巴上(R:后手靠在脸颊下颌位置,弓弦靠在下巴和鼻子上/W:后手悬空没有贴实,弦没有贴上下巴和鼻子)(S:手部评分规则参看[手部评分规则])\n"
+            "6.握弓手的手指是否放松,以在放箭之后,弓箭是否自然向前转动为标准(R:推弓手手指放松/W:手指僵直或握紧)\n"
+            "7.后手在未放箭的瞄准阶段是否有松弦现象(一般都给出R:瞄准时后手没有松滑弦 的评价/过于离谱给出W:瞄准时后手松滑弦)\n"
+            "8.双脚开步与肩同宽(R:双脚开步与肩同宽/W:双脚随意站立)(S:一般给出满分,出现极其离谱的情况可以给0/1分)\n"
+            "9.放箭时后手是否在箭射出时向后延展(随动)(R:撒放时后手顺势向后延展/W:撒放时后手定在原地或向外,向前)\n"
+            "10.放箭时前手是否放松推弓(箭射出后握弓手保持放松,弓自然转动)(R:撒放时前手放松推弓/W:撒放时前手握弓)\n"
+            "随后按要求输出JSON,给出初步评分与关键词化comment(关键词应精炼,必须使用以下的表达(符合哪项表达就使用哪项表达))\n"
+            "评分规则:\n"
+            "[手部评分规则]:三条标准:靠下巴(手指必须盖住下巴的一部分,牢牢贴合才算贴合,否则就是没有贴合),撒放向后移动,推弓(撒放后弓必须自然转动)这三条完全满足5分,少一条扣一分"
+        )
+    }
+
+    system_task_keyframes_map = {
+        "en": (
+            "\nTask:\n"
+            "You will receive a time-ordered sequence of keyframes extracted from the video (early -> late).\n"
+            "Use earlier keyframes to judge static posture/aiming quality:\n"
+            "- Body upright/vertical axis stable\n"
+            "- Head rotates sufficiently toward target\n"
+            "- Arms form a straight line / alignment is correct\n"
+            "- Rear hand firmly anchored against the chin\n"
+            "- Nose touches the bowstring (string touches chin/nose)\n"
+            "- Bow hand fingers are relaxed\n"
+            "- Before release, rear hand does NOT slip/loosen the string during aiming\n"
+            "Use later keyframes AND their changes to judge release/follow-through quality:\n"
+            "- At release, rear hand extends backward (follow-through)\n"
+            "- At release, front/bow hand relaxes and pushes/lets the bow move naturally (not grabbing)\n"
+            "Important: use the changes across later keyframes to infer release quality, not a single still.\n"
+            "Then output the required JSON with preliminary scores and a keyword-style comment.\n"
+            "Keywords MUST be concise, and MUST be selected from the following keyword library (use whichever items apply).\n"
+            "Keyword library: separate keywords with \",\".\n"
+            "1. Torso: (1) Correct: Body vertical axis is perpendicular to the ground; body is side-on to the shooting direction; "
+            "(2) Incorrect: Body center of mass is tilted/leaning; body is not rotated to a side-on stance.\n"
+            "2. Arms: (1) Correct: Front hand, front shoulder, and rear shoulder are aligned on one line; front forearm is internally rotated; "
+            "rear forearm is aligned with the arrow; "
+            "(2) Incorrect: Front shoulder has an angle and is not opened; forearm is turned outward; forearm is not aligned with the arrow.\n"
+            "3. Head: (1) Correct: Head rotates horizontally; (2) Incorrect: Head does not rotate enough.\n"
+            "4. Feet: (1) Correct: Feet stance is shoulder-width; (2) Incorrect: Feet are placed arbitrarily.\n"
+            "5. Hands: (1) Correct: Bow-hand fingers are relaxed; rear hand is anchored at the cheek/jaw area; "
+            "the bowstring touches the chin and nose; during aiming the rear hand does not slip/loosen the string; "
+            "at release the rear hand extends backward; at release the front/bow hand relaxes and pushes the bow; "
+            "(2) Incorrect: Fingers are stiff or clenched; rear hand is floating and not anchored; string does not touch chin and nose; "
+            "before/after release the rear hand slowly slides backward; at release the rear hand stays in place or moves outward/forward; "
+            "at release the front/bow hand grabs the bow.\n"
+        ),
+        "zh": (
+            "\n任务:\n"
+            "你将看到一组按时间顺序排列的关键帧(从早到晚).\n"
+            "从观察中观察并判断下面的动作,如果符合则关键词序列中直接加入对应标准后第一个括号内词库中的正确动作字符串,反之加上对应的错误动作字符串(正确动作在前,错误动作在后),并按照评分规则(每行词库后的括号)对分数进行初评,再按[生物力学特征信息],[模范]中的信息修正你的评价(如果出现)\n"
+            "1.身体是否直立(R:身体纵轴垂直于地面/W:身体重心歪斜)(可认为基本垂直的情况躯干部位给4/5分,出现歪斜给2/3分,请在给出的第一帧中判断)\n"
+            "2.身体是否转到侧对(一般给出R:身体侧向对着射箭方向 的评价,除非极其离谱,给出W:身体没有转到侧对)\n"
+            "3.头部是否转动到位(R:头部水平转动/W:头部没有转动到位)(转动到位给5分,不到位给3分,极其离谱情况给1/2分,请在给出的第一帧中判断)\n"
+            "4.手臂是否成一条直线(R:前手,前肩,后肩在一条线上,后手小臂与箭在一条线上/W:小臂与箭有角度)(后肩部有角度给2/3分,否则4/5分,请在给出的第一帧中判断)\n"
+            "5.后手是否牢牢靠在下巴上(R:后手靠在脸颊下颌位置,弓弦靠在下巴和鼻子上/W:后手悬空没有贴实,弦没有贴上下巴和鼻子)(手部评分规则参看[手部评分规则])\n"
+            "6.握弓手的手指是否放松,以在放箭之后,弓箭是否自然向前转动为标准(R:推弓手手指放松/W:手指僵直或握紧)\n"
+            "7.后手在未放箭的瞄准阶段是否有松弦现象(一般都给出R:瞄准时后手没有松滑弦 的评价/过于离谱给出W:瞄准时后手松滑弦)\n"
+            "8.双脚开步与肩同宽(R:双脚开步与肩同宽(基本满足,足部分数基本5分)/W:双脚随意站立(出现了,评价也不应低于三分))\n"
+            "9.放箭时后手是否在箭射出时向后延展(随动)(R:撒放时后手顺势向后延展/W:撒放时后手定在原地或向外,向前)\n"
+            "10.放箭时前手是否放松推弓(箭射出后握弓手保持放松,弓自然转动)(R:撒放时前手放松推弓/W:撒放时前手握弓)\n"
+            "随后按要求输出JSON,给出初步评分与关键词化comment(关键词应精炼,必须使用以下的表达(符合哪项表达就使用哪项表达))\n"
+            "评分规则:\n"
+            "[手部评分规则]:三条标准:靠下巴(手指必须盖住下巴的一部分,牢牢贴合才算贴合,否则就是没有贴合),撒放向后移动,推弓(撒放后弓必须自然转动)这三条完全满足5分,少一条扣一分,第一条请在给出的第一帧中判断,其余两条请在后面撒放的关键帧序列中比对手部位置差异来判断"
+        ),
+    }
+    
+    system_metrics_rubric_map = {
+        "en": (
+            "\nMetrics rubric (use ONLY for judgment):\n"
+            "Do NOT include any metric numeric values OR threshold numbers in 'comment'; 'comment' must contain no numbers.\n"
+            "(1) Hand-Shoulder-Elbow Angle: >=175.67 Excellent (arm=5, MUST include a positive keyword about arm straightness); "
+            "172.26-175.67 Good (arm=4, MUST include a positive keyword); "
+            "128.64-172.26 Average (if >=150 then arm=3, otherwise arm=2, MUST include a negative keyword about arm straightness); "
+            "<128.64 Poor (arm=1-2, MUST include a negative keyword).\n"
+            "(2) Hand-to-Chin Distance (two-sided target range): 232.57-274.02 Excellent "
+            "(if neither 'follow-through rear hand extends backward' nor 'bow hand relaxes and pushes' appears, hand=3; "
+            "if exactly one appears, hand=4; if both appear, hand=5; MUST include a positive keyword about hand anchored at chin; "
+            "include other hand keywords if applicable); "
+            "228.58-232.57 OR 274.02-278.02 Average "
+            "(if neither of the two release actions appears, hand=2; if one or both appear, hand=3; MUST include a negative keyword about hand anchored at chin; "
+            "include other hand keywords if applicable); "
+            "otherwise assign hand=0/1 as appropriate and use ONLY negative keywords.\n"
+            "(3) X-difference of two midpoints (two-sided target range): 61.95-87.05 Excellent "
+            "(if 68-80 then torso=5, else torso=4; MUST include ALL positive torso keywords); "
+            "53.54-61.95 OR 87.05-95.46 Good (torso=3; may include negative keywords; MUST NOT include any positive torso keywords); "
+            "45.99-53.54 OR 95.46-103.00 Average (torso=2; MUST include negative keywords); "
+            "<45.99 OR >103.00 Poor (torso=0/1; MUST include negative keywords)."
+        ),
+        "zh": (
+            "\n[生物力学特征信息](判断特定部位动作质量的关键依据,请严格根据指标数值和下面的规则修正你的评分和评估文本.下文中要给出的评价请均在词库中提取):\n"
+            "不要在 comment 中出现任何指标数值或阈值数字;comment 中不得出现数字.\n"
+            "(1)max_angle_avg:>=175.67 优秀(手臂部位5分,必须给出有关手臂伸直的正面评价(十分接近这个值也认定为优秀));172.26-175.67 良好(4分,同样必须给出正面评价);"
+            "128.64-172.26 中等(大于150度3分,小于且在区间内给2分,必须给出有关手臂伸直的负面评价);<128.64 较差(1-2,同样必须给出负面评价).\n"
+            # "(2)min_dist_avg:232.57-274.02 优秀(如果没有向后延展和推弓动作则手部为3分,具备一个4分,两个都具备5分,一定给出手贴下颌的正面评价,其他评价若具备则给出);"
+            # "228.58-232.57 或 274.02-278.02 一般(如果没有向后延展和推弓动作则手部为2分,具备一个/两个3分,一定给出手贴下颌的负面评价,其他评价若具备则给出);"
+            # "其余情况酌情给0/1分,评价均为负面评价.\n"
+            # "(3)min_x_diff_avg:61.95-87.05 优秀(68-80躯干部位5分,否则4分,必须给出全部的躯干正面评价);"
+            # "53.54-61.95 或 87.05-95.46 良好(躯干部位3分,可酌情给出负面评价,一定不出现正面评价);"
+            # "45.99-53.54 或 95.46-103.00 中等(2分,一定给出负面评价);<45.99 或 >103.00 较差(0/1分,一定给出负面评价)."
+        ),
+    }
+    
+    system_model_map = {
+        "en": (
+            ""
+        ),
+        "zh": (
+            "\n[模范]模板动作关键帧序列(请根据模范动作对你看到的动作进行严格对比评估(尤其注意头部和手部的结论更正),修正你的评估文本各部位及总体评分(例如和模范动作对比后发现动作评估有错误,就修改该错误并将对应部位评分提高,反之降低).模范动作每个部位及总体的动作质量均为满分)\n"
+            "注意,一定要仔细审查头部,手部,手臂的动作特点是否和模范动作完全相同再做出修正判断(手臂要保持直线,手部要靠位正确,有后延和推弓动作才可以满分,必须严格遵守前文提到的[评分规则])"
+        ),
+    }
+
+    if pipeline == 1:
+        system_prompt = system_base_map[lang] + system_task_video_map[lang]
+
+        video_base64 = video_to_base64(video_path)
+        completion = client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {
+                    "role": "system",
+                    "content": system_prompt,
+                },
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "video_url",
+                            "video_url": {
+                                "url": f"data:video/mp4;base64,{video_base64}"
+                            },
+                        },
+                    ],
+                }
+            ],
+            stream=True,
+        )
+
+        reasoning_content = ""
+        answer_content = ""
+        is_answering = False
+        for chunk in completion:
+            if not getattr(chunk, "choices", None):
+                continue
+            delta = chunk.choices[0].delta
+            if hasattr(delta, 'reasoning_content') and delta.reasoning_content is not None:
+                print(delta.reasoning_content, end='', flush=True)
+                reasoning_content += delta.reasoning_content
+                continue
+            if getattr(delta, "content", None):
+                if delta.content != "" and is_answering is False:
+                    print("\n" + "=" * 20 + "Complete Response" + "=" * 20 + "\n")
+                    is_answering = True
+                print(delta.content, end='', flush=True)
+                answer_content += delta.content
+        return _parse_assessment_payload(answer_content)
+
+    # pipeline 2/3/4: keyframes extraction
+    data, normalized_data = Keypoint_Extract(
+        video_path,
+        show_draw_selection=show,
+        display_wait=1,
+        image_width=1920,
+        image_height=1080,
+        draw_math_feature_points=show,
+    )
+    base_keyframes = extract_keyframes_with_ruptures_poseparts_2d(normalized_data, k=target_k + 3)
+    key_frame_lists = refine_keyframes_with_absdiff(
+        video_path=video_path,
+        keyframe_result=base_keyframes,
+        k=target_k,
+    )
+    keyframes_dict = extract_show_keyframes_by_index(video_path, key_frame_lists, show=show)
+    keyframe_image_items = list(keyframes_dict.get("openai_input_images", []) or [])
+    if not keyframe_image_items:
+        raise ValueError("No keyframes extracted; pipeline 2/3/4 requires non-empty keyframes.")
+
+    if pipeline == 2:
+        system_prompt = system_base_map[lang] + system_task_keyframes_map[lang]
+        ordering_note = (
+            "Keyframes are ordered from early to late."
+            if lang == "en"
+            else "关键帧按时间从早到晚排序."
+        )
+        raw_user_content = [{"type": "input_text", "text": ordering_note}] + keyframe_image_items
+        chat_user_content = _to_chat_content(raw_user_content)
+        resp = client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": chat_user_content},
+            ],
+        )
+        return _parse_assessment_payload(_chat_completion_output_text(resp))
+
+    math_feature = cal_math_features(keypoints_data=data,plot_angle_curve=show)
+    metrics = extract_action_features(math_feature)
+    system_prompt = system_base_map[lang] + system_task_keyframes_map[lang]     
+    if show == True:
+        print("max_angle_avg:", metrics.get("max_angle_avg"))
+        print("min_dist_avg:", metrics.get("min_dist_avg"))
+        print("min_x_diff_avg:", metrics.get("min_x_diff_avg"))
+    for k in ("max_angle_avg", "min_dist_avg", "min_x_diff_avg"):
+        if k not in metrics:
+            raise ValueError(f"Missing metric: {k}")
+
+    ordering_note = (
+        "Keyframes are ordered from early to late."
+        if lang == "en"
+        else "关键帧按时间从早到晚排序."
+    )
+    metrics_header = "\n\nMetrics:\n" if lang == "en" else "\n\n指标:\n"
+    metrics_text = (
+        metrics_header
+        + f"- max_angle_avg: {metrics['max_angle_avg']}\n"
+        # + f"- min_dist_avg: {metrics['min_dist_avg']}\n"
+        # + f"- min_x_diff_avg: {metrics['min_x_diff_avg']}"
+    )
+    if pipeline == 3:
+        p3_system_prompt = system_prompt + system_metrics_rubric_map[lang]
+        user_text = ordering_note #+ metrics_text
+        raw_user_content = [{"type": "input_text", "text": user_text}] + keyframe_image_items
+        chat_user_content = _to_chat_content(raw_user_content)
+        resp = client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {"role": "system", "content": p3_system_prompt},
+                {"role": "user", "content": chat_user_content},
+            ],
+        )
+        return _parse_assessment_payload(_chat_completion_output_text(resp))
+
+    if pipeline == 4:
+        repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+        default_template_dir = os.path.join(repo_root, "output_keyframes")
+        template_dir = template_keyframes_dir if template_keyframes_dir else default_template_dir
+        if not os.path.isabs(template_dir):
+            template_dir = os.path.abspath(os.path.join(repo_root, template_dir))
+        template_keyframe_items = _load_openai_images_from_folder(template_dir)
+
+        pipeline4_system_prompt = system_prompt + str(system_model_map.get(lang, "") or "") #+ system_metrics_rubric_map[lang]
+        if lang == "en":
+            student_note = "Student keyframes are ordered from early to late." #+ metrics_text
+            template_note = "Template keyframes are ordered from early to late."
+        else:
+            student_note = "学生关键帧按时间从早到晚排序." #+ metrics_text
+            template_note = "模范关键帧按时间从早到晚排序."
+
+        raw_user_content = (
+            [{"type": "input_text", "text": student_note}]
+            + keyframe_image_items
+            + [{"type": "input_text", "text": template_note}]
+            + template_keyframe_items
+        )
+        chat_user_content = _to_chat_content(raw_user_content)
+        resp = client.chat.completions.create(
+            model=model_name,
+            messages=[
+                {"role": "system", "content": pipeline4_system_prompt},
+                {"role": "user", "content": chat_user_content},
+            ],
+        )
+        return _parse_assessment_payload(_chat_completion_output_text(resp))
+
+_PART_ALIASES = {
+    "head": ["head", "neck", "头", "头部", "颈", "颈部"],
+    "hand": ["hand", "hands", "palm", "fingers", "wrist", "手", "手部", "手指", "手腕", "后手", "前手", "握弓手", "推弓手"],
+    "arm": ["arm", "arms", "elbow", "shoulder", "forearm", "upperarm", "手臂", "臂", "肩", "肩部", "肘", "肘部", "小臂", "大臂", "共线", "一条线", "直线"],
+    "torso": ["torso", "body", "trunk", "core", "posture", "躯干", "身体", "中轴", "重心", "背部", "上身"],
+    "foot": ["foot", "feet", "leg", "stance", "脚", "足", "足部", "双脚", "步态", "站姿", "开步", "与肩同宽"],
+}
+
+def _normalize_language_qa(language: str) -> str:
+    language_norm = str(language or "en").strip().lower()
+    if language_norm in ("zh", "zh-cn", "zh_hans", "cn", "chinese", "中文"):
+        return "zh"
+    if language_norm in ("en", "en-us", "english"):
+        return "en"
+    return "zh"
+
+
+def _normalize_text_for_match(text: str) -> str:
+    return re.sub(r"[\s_\-]+", "", str(text or "").strip().lower())
+
+
+def _normalize_part_name(part: str):
+    part_norm = _normalize_text_for_match(part)
+    for canon_part, aliases in _PART_ALIASES.items():
+        for alias in aliases:
+            if part_norm == _normalize_text_for_match(alias):
+                return canon_part
+    return None
+
+
+def _sanitize_question_text(question: str) -> str:
+    text = str(question or "")
+    text = re.sub(r"[^A-Za-z0-9\u4e00-\u9fff\s]", " ", text)
+    text = re.sub(r"\s+", " ", text).strip()
+    if not text:
+        raise ValueError("Question is empty after sanitization.")
+    return text
+
+
+def _extract_json_object_text(raw_text: str) -> str:
+    text = str(raw_text or "").strip()
+    if not text:
+        raise ValueError("LLM output is empty.")
+    decoder = json.JSONDecoder()
+    for i, ch in enumerate(text):
+        if ch != "{":
+            continue
+        try:
+            parsed, end = decoder.raw_decode(text[i:])
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict):
+            return text[i : i + end]
+    raise ValueError("No valid JSON object found in LLM output.")
+
+
+def _chat_completion_output_text(resp) -> str:
+    if resp is None or not getattr(resp, "choices", None):
+        raise ValueError("Chat completion returned empty response.")
+    message = getattr(resp.choices[0], "message", None)
+    content = getattr(message, "content", None)
+    if isinstance(content, str):
+        return content.strip()
+    if isinstance(content, list):
+        parts = []
+        for item in content:
+            if isinstance(item, dict):
+                txt = item.get("text")
+                if txt:
+                    parts.append(str(txt))
+            else:
+                txt = getattr(item, "text", None)
+                if txt:
+                    parts.append(str(txt))
+        return "".join(parts).strip()
+    return str(content or "").strip()
+
+
+def _split_question_with_llm(question_clean: str, language: str, model: str) -> dict:
+    lang = _normalize_language_qa(language)
+    client = OpenAI(
+        api_key=os.getenv("ALI_API_KEY"),
+        base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
+    )
+    system_prompt_map = {
+       "zh": (
+            "你是一个专业的问答问题拆分器，面向“视频中人的射箭动作分析”场景."
+            "你的任务是把用户问题字符串整理提炼为后续大模型可以正确理解的两类内容并识别下文提到的两个特征,并返回JSON对象字符串.\n"
+            "JSON字段固定为technical_part(内容1), knowledge_part(内容2), technical_scope(特征1), target_parts(特征2)\n"
+            "1)technical_part:与视频中人物射箭动作技术相关的提问(字符串)\n"
+            "2)knowledge_part:与射箭相关的通用知识内容(如规则、器材、术语解释、训练方法原理、比赛常识等)(字符串).\n"
+            "特征1的所有可能结果为:overall/parts/none.判断准则如下:\n"
+            "- overall:技术问题主要针对整体动作或完整流程.\n"
+            "- parts:技术问题主要针对具体身体部位.\n"
+            "- none:问题不涉及视频动作技术.\n"
+            "如特征1的结果为parts,则对特征2进行识别,找出技术问题主要针对下列哪个或哪些身体部位:head/hand/arm/torso/foot,结果应该为列表.若特征1为其他结果则返回空列表."
+            "根据问题指向选择一个或多个部位；否则 target_parts 为空列表."
+        ),
+        "en": (
+            "You are a QA question splitter. "
+            "Output JSON object string only, with keys: technical_part, knowledge_part, technical_scope, target_parts. "
+            "technical_scope must be one of overall/parts/none. "
+            "target_parts can only include head/hand/arm/torso/foot. "
+            "Use empty string/list if a field is not applicable."
+        ),
+    }
+    user_prompt_map = {
+        "zh": f"问题:{question_clean}\n",
+        "en": f"Question: {question_clean}\nSplit the question and return JSON only.",
+    }
+
+    resp = client.chat.completions.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": system_prompt_map[lang]},
+            {"role": "user", "content": user_prompt_map[lang]},
+        ],
+        temperature=0.1,
+    )
+    raw_text = _chat_completion_output_text(resp)
+    payload = json.loads(_extract_json_object_text(raw_text))
+    if not isinstance(payload, dict):
+        raise ValueError("Question split output is not a JSON object.")
+
+    technical_part = str(payload.get("technical_part", "") or "").strip()
+    knowledge_part = str(payload.get("knowledge_part", "") or "").strip()
+    technical_scope = str(payload.get("technical_scope", "none") or "none").strip().lower()
+    if technical_scope not in {"overall", "parts", "none"}:
+        technical_scope = "none"
+
+    target_parts_raw = payload.get("target_parts", [])
+    if not isinstance(target_parts_raw, list):
+        target_parts_raw = []
+    target_parts = []
+    for part in target_parts_raw:
+        canon_part = _normalize_part_name(str(part))
+        if canon_part and canon_part not in target_parts:
+            target_parts.append(canon_part)
+    return {
+        "technical_part": technical_part,
+        "knowledge_part": knowledge_part,
+        "technical_scope": technical_scope,
+        "target_parts": target_parts,
+    }
+
+def _map_keyword_to_part(keyword: str, language: str):
+    _ = _normalize_language_qa(language)
+    keyword_norm = _normalize_text_for_match(keyword)
+    if not keyword_norm:
+        return None
+
+    part_order = ["arm", "hand", "torso", "head", "foot"]
+    best_part = None
+    best_count = 0
+    best_first_pos = None
+
+    for part in part_order:
+        part_count = 0
+        part_first_pos = None
+        for alias in _PART_ALIASES.get(part, []):
+            alias_norm = _normalize_text_for_match(alias)
+            if not alias_norm:
+                continue
+            hit_count = keyword_norm.count(alias_norm)
+            if hit_count <= 0:
+                continue
+            part_count += hit_count
+            hit_pos = keyword_norm.find(alias_norm)
+            if hit_pos >= 0 and (part_first_pos is None or hit_pos < part_first_pos):
+                part_first_pos = hit_pos
+
+        if part_count <= 0:
+            continue
+
+        if best_part is None:
+            best_part = part
+            best_count = part_count
+            best_first_pos = part_first_pos
+            continue
+
+        if part_count > best_count:
+            best_part = part
+            best_count = part_count
+            best_first_pos = part_first_pos
+            continue
+
+        if part_count == best_count:
+            current_pos = part_first_pos if part_first_pos is not None else 10**9
+            best_pos = best_first_pos if best_first_pos is not None else 10**9
+            if current_pos < best_pos:
+                best_part = part
+                best_first_pos = part_first_pos
+
+    if best_part:
+        return best_part
+    return None
+
+def _filter_keywords_for_parts(keywords: list[str], parts: list[str], language: str) -> list[str]:
+    target_parts = []
+    for part in parts:
+        canon_part = _normalize_part_name(part)
+        if canon_part and canon_part not in target_parts:
+            target_parts.append(canon_part)
+    if not target_parts:
+        return []
+
+    filtered = []
+    seen = set()
+    for kw in keywords:
+        kw_str = str(kw).strip()
+        if not kw_str:
+            continue
+        part = _map_keyword_to_part(kw_str, language)
+        if part in target_parts and kw_str not in seen:
+            filtered.append(kw_str)
+            seen.add(kw_str)
+    return filtered
+
+
+# def _retrieve_by_keywords(
+#     keywords: list[str],
+#     session,
+#     embedding_model: str,
+#     top_k_per_keyword: int,
+# ) -> list[str]:
+#     from RAG.Knowledge_Database.AI_dbmanager import KnowledgeDB
+
+#     db = KnowledgeDB(session=session)
+#     snippets = []
+#     seen = set()
+#     for kw in keywords:
+#         kw_text = str(kw).strip()
+#         if not kw_text:
+#             continue
+#         try:
+#             search_results = db.search(
+#                 kw_text,
+#                 embed_fn=get_embedding,
+#                 model_name=embedding_model,
+#                 top_k=top_k_per_keyword,
+#             )
+#         except Exception as e:
+#             print(f"[QA] keyword retrieval failed for '{kw_text}': {e}")
+#             continue
+
+#         for result in search_results:
+#             chunk = result[0] if isinstance(result, tuple) else result
+#             chunk_text = getattr(chunk, "text", None)
+#             if chunk_text is None and isinstance(chunk, str):
+#                 chunk_text = chunk
+#             chunk_text = str(chunk_text or "").strip()
+#             if not chunk_text or chunk_text in seen:
+#                 continue
+#             snippets.append(chunk_text)
+#             seen.add(chunk_text)
+#     return snippets
+
+
+def _retrieve_knowledge_by_question(
+    question_text: str,
+    session,
+    embedding_model: str,
+    title_top_k: int,
+    chunk_top_k: int,
+) -> list[str]:
+    from RAG.Knowledge_Database.AI_dbmanager import KnowledgeDB
+
+    db = KnowledgeDB(session=session)
+    try:
+        search_results = db.search_knowledge_two_stage(
+            str(question_text or "").strip(),
+            embed_fn=get_embedding,
+            model_name=embedding_model,
+            title_top_k=title_top_k,
+            chunk_top_k=chunk_top_k,
+        )
+    except Exception as e:
+        print(f"[QA] knowledge retrieval failed: {e}")
+        return []
+
+    snippets = []
+    seen = set()
+    for result in search_results:
+        chunk = result[0] if isinstance(result, tuple) else result
+        chunk_text = getattr(chunk, "text", None)
+        if chunk_text is None and isinstance(chunk, str):
+            chunk_text = chunk
+        chunk_text = str(chunk_text or "").strip()
+        if not chunk_text or chunk_text in seen:
+            continue
+        snippets.append(chunk_text)
+        seen.add(chunk_text)
+    return snippets
+
+def _compress_eval_text(
+    eval_text: str,
+    scope: str,
+    target_parts: list[str],
+    language: str,
+    model: str,
+) -> str:
+    text = str(eval_text or "").strip()
+    if not text:
+        return ""
+
+    scope_norm = str(scope or "overall").strip().lower()
+    target_part_list = []
+    for part in target_parts or []:
+        canon_part = _normalize_part_name(part)
+        if canon_part and canon_part not in target_part_list:
+            target_part_list.append(canon_part)
+    target_part_text = ", ".join(target_part_list) if target_part_list else "(none)"
+
+    lang = _normalize_language_qa(language)
+    client = OpenAI(
+        api_key=os.getenv("ALI_API_KEY"),
+        base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
+    )
+    system_prompt_map = {
+        "zh": (
+            "你是射箭评估文本提取与语言修缮助手.\n"
+            "若 scope=parts: 只提取与 target_parts 对应部位相关的评价信息,忽略其他部位.\n"
+            "若 scope=overall: 提炼整体动作评价信息.\n"
+            "将结果压缩成1-2句动作要点,不要添加原文之外的信息.\n"
+            "若 scope=parts 且原文无对应部位信息,返回空字符串.\n"
+            "输出仅为纯文本."
+        ),
+        "en": (
+            "You are an archery-evaluation extraction and summarization assistant.\n"
+            "If scope=parts: extract ONLY evaluation content related to target_parts and ignore other parts.\n"
+            "If scope=overall: summarize the overall action evaluation.\n"
+            "Compress into 1-2 short action-focused sentences without adding extra facts.\n"
+            "If scope=parts and no relevant information exists, return an empty string.\n"
+            "Output plain text only."
+        ),
+    }
+    user_prompt_map = {
+        "zh": (
+            f"scope: {scope_norm}\n"
+            f"target_parts: {target_part_text}\n"
+            f"原文: {text}\n"
+        ),
+        "en": (
+            f"scope: {scope_norm}\n"
+            f"target_parts: {target_part_text}\n"
+            f"source: {text}\n"
+            "Extract first, then summarize."
+        ),
+    }
+    try:
+        resp = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": system_prompt_map[lang]},
+                {"role": "user", "content": user_prompt_map[lang]},
+            ],
+            temperature=0.2,
+        )
+        summary = _chat_completion_output_text(resp).strip()
+        if summary:
+            return summary
+    except Exception as e:
+        print(f"[QA] eval text compression failed: {e}")
+
+    if scope_norm == "parts":
+        return ""
+    return text[:220]
+
+
+def _compose_qa_answer(
+    question_tech: str,
+    tech_summary: str,
+    keywords: list[str],
+    knowledge_snippets: list[str],
+    language: str,
+    tech_suggestions = None,
+    knowledge_question: str = "",
+    model: str = "qwen-plus",
+) -> str:
+    
+    lang = _normalize_language_qa(language)
+    tech_summary = str(tech_summary or "").strip()
+    keywords = [str(k).strip() for k in (keywords or []) if str(k).strip()]
+    tech_suggestions = [str(s).strip() for s in (tech_suggestions or []) if str(s).strip()]
+    knowledge_question = str(knowledge_question or "").strip()
+    knowledge_snippets = [str(s).strip() for s in (knowledge_snippets or []) if str(s).strip()]
+
+    client = OpenAI(
+        api_key=os.getenv("ALI_API_KEY"),
+        base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
+    )
+    keyword_text = ", ".join(keywords[:12]) if keywords else "(none)"
+    suggestion_seed = ";".join(tech_suggestions[:8]) if tech_suggestions else "(none)"
+    snippet_text = "\n".join([f"- {s}" for s in knowledge_snippets[:8]]) if knowledge_snippets else "- (empty)"
+
+    if question_tech != "":
+        if knowledge_question != "":
+            system_prompt_map = {
+                "zh": (
+                    "你是射箭问答整合助手.\n"
+                    "你需要在一次回答中完成三个模块并最终拼接输出,保证语义一致性,逻辑严谨性��专业性:\n"
+                    "1) 技术动作分析: 按照tech_summary给出当前针对部位的技术动作分析(尽可能使用原句进行分析)\n"
+                    "2) 改进建议: 依据keywords对question_tech给出改进建议,可使用你的通用知识.\n"
+                    "3) 知识问题回答: 仅依据knowledge_snippets中能够回答knowledge_question的片段进行回答"
+                    ",不得引入 snippets 外信息,你的任务是将零碎的信息整合成段落.\n"
+                    "若 snippets 不足,必须写“根据已检索片段无法确定”.\n"
+                    "最后直接输出三行,格式固定为:\n"
+                    "对您问题中涉及的技术动作进行分析:...\n针对不足之处的改进建议:...\n关于知识问题的回答:...\n"
+                    "不要输出中间推理、不要分点编号、不要额外标题."
+                ),
+                "en": (
+                    "You are an archery QA integrator.\n"
+                    "In ONE response, complete three modules and merge them with semantic consistency:\n"
+                    "1) Technical Action Analysis: based on tech_summary.\n"
+                    "2) Improvement Advice: based on question + keywords; general knowledge is allowed.\n"
+                    "3) Knowledge Answer: answer knowledge_question using ONLY knowledge_snippets.\n"
+                    "If snippets are insufficient, you must output 'Cannot determine from retrieved snippets.'\n"
+                    "Return exactly three lines:\n"
+                    "Technical Action Analysis:...\nImprovement Advice:...\nKnowledge Answer:...\n"
+                    "No extra headings or reasoning."
+                ),
+            }
+        else:
+            system_prompt_map = {
+                "zh": (
+                    "你是射箭问答整合助手.\n"
+                    "你需要在一次回答中完成两个模块并最终拼接输出,保证语义一致性,逻辑严谨性和专业性:\n"
+                    "1) 技术动作分析: 按照tech_summary给出当前针对部位的技术动作分析(尽可能使用原句进行分析)\n"
+                    "2) 改进建议: 依据 question_tech + keywords 给出改进建议,可使用你的通用知识.\n"
+                    ",不得引入snippets外信息,你的任务是将零碎的信息整合成段落.\n"
+                    "若 snippets 不足,必须写“根据已检索片段无法确定”.\n"
+                    "最后直接输出两行,格式固定为:\n"
+                    "对您问题中涉及的技术动作进行分析:...\n针对不足之处的改进建议:...\n"
+                    "不要输出中间推理、不要分点编号、不要额外标题."
+                ),
+                "en": (
+                    "You are an archery QA integrator.\n"
+                    "In ONE response, complete three modules and merge them with semantic consistency:\n"
+                    "1) Technical Action Analysis: based on tech_summary.\n"
+                    "2) Improvement Advice: based on question + keywords; general knowledge is allowed.\n"
+                    "3) Knowledge Answer: answer knowledge_question using ONLY knowledge_snippets.\n"
+                    "If snippets are insufficient, you must output 'Cannot determine from retrieved snippets.'\n"
+                    "Return exactly two lines:\n"
+                    "Technical Action Analysis:...\nImprovement Advice:...\n"
+                    "No extra headings or reasoning."
+                ),
+            }
+    else:
+        system_prompt_map = {
+                "zh": (
+                    "你是射箭问答整合助手.\n"
+                    "你需要在一次回答中完成一个模块并最终拼接输出,保证语义一致性,逻辑严谨性和专业性:\n"
+                    "1) 知识问题回答: 仅依据knowledge_snippets中能够回答knowledge_question的片段进行回答"
+                    ",不得引入snippets外信息,你的任务是将零碎的信息整合成段落.\n"
+                    "若 snippets 不足,必须写“根据已检索片段无法确定”.\n"
+                    "最后直接输出一行,格式固定为:\n"
+                    "关于知识问题的回答:\n"
+                    "不要输出中间推理、不要分点编号、不要额外标题."
+                ),
+                "en": (
+                    "You are an archery QA integrator.\n"
+                    "In ONE response, complete three modules and merge them with semantic consistency:\n"
+                    "1) Technical Action Analysis: based on tech_summary.\n"
+                    "2) Improvement Advice: based on question + keywords; general knowledge is allowed.\n"
+                    "3) Knowledge Answer: answer knowledge_question using ONLY knowledge_snippets.\n"
+                    "If snippets are insufficient, you must output 'Cannot determine from retrieved snippets.'\n"
+                    "Return exactly two lines:\n"
+                    "Technical Action Analysis:...\nImprovement Advice:...\n"
+                    "No extra headings or reasoning."
+                ),
+            }
+    user_prompt_map = {
+        "zh": (
+            f"question_tech: {question_tech}\n"
+            f"tech_summary: {tech_summary if tech_summary else '(none)'}\n"
+            f"keywords: {keyword_text}\n"
+            # f"tech_suggestions_seed: {suggestion_seed}\n"
+            f"knowledge_question: {knowledge_question if knowledge_question else '(none)'}\n"
+            f"knowledge_snippets:\n{snippet_text}\n"
+            "请按系统要求一次性完成并拼接."
+        ),
+        "en": (
+            f"question_tech: {question_tech}\n"
+            f"tech_summary: {tech_summary if tech_summary else '(none)'}\n"
+            f"keywords: {keyword_text}\n"
+            # f"tech_suggestions_seed: {suggestion_seed}\n"
+            f"knowledge_question: {knowledge_question if knowledge_question else '(none)'}\n"
+            f"knowledge_snippets:\n{snippet_text}\n"
+            "Complete and merge in one pass following the required format."
+        ),
+    }
+    try:
+        resp = client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": system_prompt_map[lang]},
+                {"role": "user", "content": user_prompt_map[lang]},
+            ],
+            temperature=0.3,
+        )
+        merged = _chat_completion_output_text(resp).strip()
+        if merged:
+            return merged
+    except Exception as e:
+        print(f"[QA] one-shot compose failed: {e}")
+
+    if lang == "zh":
+        knowledge_fallback = "根据已检索片段无法确定." if knowledge_question else "无知识型问题."
+        advice_fallback = (
+            ";".join(tech_suggestions[:8])
+            if tech_suggestions
+            else "当前暂无建议库,可先围绕关键词逐项练习并复查动作稳定性."
+        )
+        return "\n".join(
+            [
+                f"技术动作分析:{tech_summary if tech_summary else '暂无可用技术动作摘要.'}",
+                f"改进建议:{advice_fallback}",
+                f"知识问题回答:{knowledge_fallback}",
+            ]
+        )
+
+    knowledge_fallback = "Cannot determine from retrieved snippets." if knowledge_question else "No knowledge question."
+    advice_fallback = (
+        "; ".join(tech_suggestions[:8])
+        if tech_suggestions
+        else "No suggestion DB is available yet; focus on keyword-related drills and re-check movement stability."
+    )
+    return "\n".join(
+        [
+            f"Technical Action Analysis: {tech_summary if tech_summary else 'No technical summary available.'}",
+            f"Improvement Advice: {advice_fallback}",
+            f"Knowledge Answer: {knowledge_fallback}",
+        ]
+    )
+#TODO Query逻辑写完
+def answer_archery_question(
+    keywords: list[str],
+    evaluation_text: str,
+    question: str,
+    tech_session,
+    knowledge_session,
+    language: str = "zh",
+    embedding_model: str = "ali-text-embedding-v4",
+    classifier_model: str = "qwen-plus",
+    summarizer_model: str = "qwen-plus",
+    top_k_knowledge_title: int = 5,
+    top_k_knowledge_chunks: int  = 8,
+    show : bool = False
+) -> str:
+    """
+    基于“技术评估上下文 + 静态知识库”生成射箭问答回复。
+
+    流程概览:
+    1) 规范化语言与输入问题
+    2) 用 LLM 拆分问题(技术细节 / 知识问答 / 作用范围)
+    3) 按范围汇总评估文本并提取技术动作摘要
+    4) 对知识子问题做向量检索补充
+    5) 基于关键词生成改进建议 + 基于知识片段回答知识问题
+    6) 统一拼接为最终自然语言答案
+    """
+    try:
+        # 统一语言标识并清洗关键词，避免空白词污染检索。
+        lang = _normalize_language_qa(language) #return zh/en
+        clean_keywords = [str(k).strip() for k in (keywords or []) if str(k).strip()]
+        try:
+            # 对用户问题做基础净化(去噪/长度与格式保护)。
+            question_clean = _sanitize_question_text(question)#去所有特殊符号和标点，转换成空格
+        except Exception:
+            return "问题为空或无有效内容,请提供具体问题." if lang == "zh" else "The question is empty after sanitization. Please provide a concrete question."
+
+        # 当问题拆分失败时，退化为“全部按技术问题处理”的保底策略。
+        split_fallback = {
+            "technical_part": question_clean,
+            "knowledge_part": "",
+            "technical_scope": "overall",
+            "target_parts": [],
+        }
+        try:
+            split_result = _split_question_with_llm(
+                question_clean=question_clean,
+                language=lang,
+                model=classifier_model,
+            )
+            if show == True:
+                print(split_result)
+        except Exception as e:
+            print(f"[QA] question split failed, use fallback: {e}")
+            split_result = split_fallback
+
+        # 解析拆分结果并约束 scope 枚举值，防止异常输出影响后续分支。
+        technical_part = str(split_result.get("technical_part", "") or "").strip()
+        knowledge_part = str(split_result.get("knowledge_part", "") or "").strip()
+        technical_scope = str(split_result.get("technical_scope", "none") or "none").strip().lower()
+        if technical_scope not in {"overall", "parts", "none"}:
+            technical_scope = "none"
+
+        # 部位名称做标准化并去重，用于“分部位技术检索”。
+        target_parts_raw = split_result.get("target_parts", [])
+        target_parts = []
+        if isinstance(target_parts_raw, list):
+            for part in target_parts_raw:
+                canon_part = _normalize_part_name(str(part)) #match the correct name for each part
+                if canon_part and canon_part not in target_parts:
+                    target_parts.append(canon_part)
+
+        tech_summary = ""
+        tech_suggestions = None
+        advice_keywords = clean_keywords
+
+        if technical_part:
+            if technical_scope == "parts" and target_parts:
+                # 分部位问题: 优先使用部位过滤后的关键词，并抽取对应部位评估摘要。
+                scoped_keywords = _filter_keywords_for_parts(clean_keywords, target_parts, lang)
+                if not scoped_keywords:
+                    # 过滤后为空时回退到原始关键词，避免检索结果为空。
+                    scoped_keywords = clean_keywords
+                advice_keywords = scoped_keywords
+                tech_summary = _compress_eval_text(
+                    eval_text=evaluation_text,
+                    scope="parts",
+                    target_parts=target_parts,
+                    language=lang,
+                    model=summarizer_model,
+                )
+            else:
+                # 整体技术问题: 使用整体评估摘要 + 全关键词建议生成。
+                tech_summary = _compress_eval_text(
+                    eval_text=evaluation_text,
+                    scope="overall",
+                    target_parts=[],
+                    language=lang,
+                    model=summarizer_model,
+                )
+        knowledge_snippets = []
+        if knowledge_part:
+            # 知识型子问题单独检索，避免与技术建议混淆。
+            knowledge_snippets = _retrieve_knowledge_by_question(
+                question_text=knowledge_part,
+                session=knowledge_session,
+                embedding_model=embedding_model,
+                title_top_k=top_k_knowledge_title,
+                chunk_top_k=top_k_knowledge_chunks,
+            )
+            if show == True:
+                print(knowledge_snippets)
+
+        # 汇总技术摘要、关键词建议、知识片段问答，生成最终文本。
+        return _compose_qa_answer(
+            question_tech=technical_part,
+            tech_summary=tech_summary,
+            keywords=advice_keywords,
+            tech_suggestions=tech_suggestions,
+            knowledge_question=knowledge_part,
+            knowledge_snippets=knowledge_snippets,
+            language=lang,
+            model=summarizer_model,
+        )
+        
+    except Exception as e:
+        print(f"[QA] answer_archery_question failed: {e}")
+        lang = _normalize_language_qa(language)
+        if lang == "zh":
+            return "问答处理失败,请稍后重试."
+        return "QA processing failed. Please try again later."
+
+def get_response(
+    keywords: list[str],
+    score_dict: dict,
+    retrieved_snippets: list[str],
+    language: str = "en",
+) -> str:
+    """
+    Generate final assessment text from:
+    - keyword sequence
+    - preliminary score dictionary
+    - retrieved database snippets
+    """
+    # client = OpenAI(
+    #     base_url="https://openrouter.ai/api/v1",
+    #     api_key=os.environ.get("OPENROUTER_API_KEY"),
+    # )
+    
+    client = OpenAI(
+        api_key=os.getenv("ALI_API_KEY"),
+        base_url="https://dashscope.aliyuncs.com/compatible-mode/v1"
+    )
+
+    language_norm = str(language or "en").strip().lower()
+    if language_norm in ("zh", "zh-cn", "zh_hans", "cn", "chinese", "中文"):
+        lang = "zh"
+    elif language_norm in ("en", "en-us", "english"):
+        lang = "en"
+    else:
+        raise ValueError("language must be 'en' or 'zh'")
+
+    if not isinstance(keywords, (list, tuple)):
+        raise ValueError("keywords must be a list of strings")
+    if not isinstance(score_dict, dict):
+        raise ValueError("score_dict must be a dict")
+    if not isinstance(retrieved_snippets, (list, tuple)):
+        raise ValueError("retrieved_snippets must be a list of strings")
+
+    def _normalize_snippet_item(item) -> str:
+        if isinstance(item, str):
+            return item.strip()
+        if isinstance(item, dict):
+            for key in ("text", "content", "chunk", "document", "comment"):
+                value = item.get(key)
+                if value:
+                    return str(value).strip()
+            return json.dumps(item, ensure_ascii=False)
+        return str(item).strip()
+
+    cleaned_keywords = [str(k).strip() for k in keywords if str(k).strip()]
+    cleaned_snippets = [_normalize_snippet_item(s) for s in retrieved_snippets]
+    cleaned_snippets = [s for s in cleaned_snippets if s]
+
+    system_instruction_map = {
+        "en": (
+            "You are a professional archery evaluation coach.\n"
+            "Use only user-provided inputs: keyword sequence, score dictionary, and retrieved DB snippets.\n"
+            "Do not search or use external knowledge.\n"
+            "Output must be plain assessment text only (not JSON, no markdown, no code block).\n"
+            "Write a coherent professional summary and improvement suggestions.\n"
+            "Use this format: This student's correct actions are: ... This student's incorrect actions are: ...\n"
+            "Prefer retrieved DB snippets when semantically consistent with keywords and scores.\n"
+            "Avoid contradictions and keep length around 550 characters."
+        ),
+        "zh": (
+            "你是一名专业的射箭评估教练.\n"
+            "只能使用用户提供的输入:关键词序列、评估分数字典、数据库检索片段.\n"
+            "不得搜索或引用外部知识,你的任务是根据输入内容识别表述的一致性并进行筛选、对输入信息进行拼接、生成流畅的评估文本.\n"
+            "输出必须是纯文本评估结果(不要 含有JSON、Markdown 或代码块,也不要含有特殊字符).\n"
+            "写出连贯、专业的动作质量评估与改进建议.\n"
+            "确保最终评估文本中不存在语义完全相反(自相矛盾)的语句后再进行输出.\n"
+            "只做评估工作,不进行额外的解释或说明(根据数据库中语句......可得到等类似的措辞不要出现,也不要出现数据)\n"
+            "行文格式必须为:这位同学的正确动作如下: ... 这位同学的错误动作如下: ...\n"
+            "当数据库片段与关键词及分数字典语义一致时优先使用数据库片段表达,如不一致则使用关键词序列中的表达.\n"
+            "文本整体长度控制在约 550 字符."
+            "数据库检索得到的文本片段并非完全正确,请仔细甄别,只使用与关键词序列语义相同的关键词."
+            "注意,射箭人的动作并非一定有缺点,如果关键词序列中无负面评价,“这位同学的错误动作如下：”冒号后不填写任何东西即可.\n"
+        ),
+    }
+
+    system_instruction = system_instruction_map[lang]
+    keyword_text = "\n".join([f"- {k}" for k in cleaned_keywords]) if cleaned_keywords else "- (empty)"
+    snippet_text = "\n".join([f"- {s}" for s in cleaned_snippets]) if cleaned_snippets else "- (empty)"
+    score_json = json.dumps(score_dict, ensure_ascii=False, indent=2)
+
+    user_prompt_map = {
+        "en": (
+            "Keywords:\n"
+            f"{keyword_text}\n\n"
+            "Score dictionary(Head, hands, feet, arms, torso and overall movement quality assessment):\n"
+            f"{score_json}\n\n"
+            "Retrieved database snippets:\n"
+            f"{snippet_text}\n\n"
+            "Return only the final assessment text string."
+        ),
+        "zh": (
+            "关键词序列:\n"
+            f"{keyword_text}\n\n"
+            "评估分数字典(头部,手部,足部,手臂,躯干及整体动作质量评分):\n"
+            f"{score_json}\n\n"
+            "数据库检索片段:\n"
+            f"{snippet_text}\n\n"
+            "只返回最终评估文本字符串."
+        ),
+    }
+
+    messages = [
+        {"role": "system", "content": system_instruction},
+        {"role": "user", "content": user_prompt_map[lang]},
+    ]
+
+    answer_content = ""
+    try:
+        completion = client.chat.completions.create(
+            model="qwen-plus",
+            messages=messages,
+            stream=True,
+            temperature=0.7,
+        )
+
+        print("\n" + "=" * 20 + " Generating Assessment Text " + "=" * 20)
+        is_answering = False
+
+        for chunk in completion:
+            delta = chunk.choices[0].delta
+
+            if hasattr(delta, "reasoning_content") and delta.reasoning_content:
+                if not is_answering:
+                    print(f"\n[Thinking]: {delta.reasoning_content}", end="", flush=True)
+
+            if hasattr(delta, "content") and delta.content:
+                if not is_answering:
+                    print("\n" + "-" * 45)
+                    is_answering = True
+                print(delta.content, end="", flush=True)
+                answer_content += delta.content
+
+        print("\n" + "=" * 45)
+
+    except Exception as e:
+        print(f"API call error: {e}")
+        return f"Error: {e}"
+
+    return answer_content.strip()
+
+def get_embedding_languagebind_text(texts):
+    """Get text embeddings using LanguageBind model"""
+    texts = texts if isinstance(texts, list) else [texts]
+    device = torch.device("cuda:0")
+    
+    tokenizer = LanguageBindImageTokenizer.from_pretrained(
+        'lb203/LanguageBind_Image',
+        cache_dir='./cache_dir/tokenizer_cache_dir'
+    )
+    
+    model = LanguageBind(clip_type={'image': 'LanguageBind_Image'},
+                         cache_dir='./cache_dir').to(device)
+    model.eval()
+    
+    inputs = {
+        'language': to_device(tokenizer(texts, max_length=77, 
+                                        padding='max_length', 
+                                        truncation=True, 
+                                        return_tensors='pt'), 
+                                        device)
+    }
+    
+    with torch.no_grad():
+        embeddings = model(inputs)['language']
+    return embeddings.cpu().numpy()
+
+def get_embedding_languagebind_video(video_path):
+    """Get video embeddings using LanguageBind model"""
+    video_path = video_path if isinstance(video_path, list) else [video_path]
+    clip_type = {
+        'video': 'LanguageBind_Video_FT',
+    }
+    model = LanguageBind(clip_type=clip_type)
+    model = model.to(device)
+    model.eval()
+    modality_transform = {c: transform_dict[c](model.modality_config[c]) for c in clip_type.keys()}
+
+    inputs = {
+        'video': to_device(modality_transform['video'](video_path), device)
+    }
+    
+    with torch.no_grad():
+        embeddings = model(inputs)['video']
+    return embeddings.cpu().numpy()
+
+def construct_complex_prompt(scores: Dict, prompt: str, comment_text: List[str]):
+    """
+    Construct a complex prompt containing structured scores, retrieved comments, and user instructions
+    """
+    scores_str = json.dumps(scores, indent=2, ensure_ascii=False)
+
+    formatted_comments = "\n".join([f"{i+1}. {text}" for i, text in enumerate(comment_text)])
+    print(formatted_comments)
+
+    final_content = f"""Please evaluate the archer's posture based on the following information:
+    1. Movement Scores for Each Body Part of Athletes (json):
+    {scores_str}
+    2. Evaluation keywords and phrases provided by professional coaches retrieved from the database:
+    {formatted_comments}
+    """
+    return final_content
+

RAG/Knowledge_Database/languagebind_main/1/1

@@ -0,0 +1 @@
+1

RAG/Knowledge_Database/languagebind_main/DATASETS.md

@@ -0,0 +1,66 @@
+## Sample data
+We are releasing sample data here so that individuals who are interested can further modify the code to train it on their own data, which includes videos, text from various sources, depth, and infrared.
+
+<div align="center">
+<table border="1" width="100%">
+    <tr align="center">
+        <th></th><th>Baidu Yun</th><th>Google Cloud</th><th>Peking University Yun</th>
+    </tr>
+    <tr align="center">
+        <td>DATA</td><td><a href="https://pan.baidu.com/s/1MnQUO6xrMPE5HAwveAdtZA?pwd=5ug9">Link</a></td><td><a href="https://drive.google.com/file/d/1p7y_0H3c84VbWpI-zx_m_mgn84uTZTdO/view?usp=drive_link">Link</a></td><td><a href="https://disk.pku.edu.cn:443/link/B6BDBDDCC616D47126DD0FF568CAF6CD">Link</a></td>
+    </tr>
+    <tr align="center">
+        <td>ANNOTATION</td><td><a href="https://pan.baidu.com/s/1uxxx_67VWy25q7CDilLsHA?pwd=37j3">Link</a></td><td><a href="https://drive.google.com/file/d/1WWVkt9LdbGK0VeQh-g7xy1gUGBwzwVah/view?usp=drive_link">Link</a></td><td><a href=https://disk.pku.edu.cn:443/link/67D836DE504E96457554455A597DC57F"">Link</a></td>
+    </tr>
+</table>
+</div>
+
+## VIDAL-10M
+
+### Text and Video
+Due to policy restrictions, we are unable to directly release the videos. However, we provide the YouTube IDs, which can be used to download the videos independently. All textual sources and YouTube IDs can be downloaded from [Google Disk](https://drive.google.com/file/d/1qgm3rO9JugazLJ6KRsAKZfLIagHu3PJ-/view?usp=sharing) or [Baidu Disk](https://pan.baidu.com/s/13gY-IcFSFIuDZ-q0hMTx0g?pwd=gum9).
+
+The organization format of `ANNOTATION` is as follows:
+```Bash
+{
+  "ImkVYKWqlDU": {
+    "folder": "coco_vat_9",
+    "mplug": "This video describes a group of scuba divers rolling backwards off a boat while playing an instrument. They are having fun and enjoying their time in the water.",
+    "polish_mplug": "scuba divers are seen rolling backwards off a boat while playing an instrument, displaying enjoyment and having a good time in the water.",
+    "ofa": [
+      " a man in a wet suit and a helmet on a boat",
+      " a man in a scuba suit on a boat",
+      " a person in a boat holding a diver helmet",
+      " a man in a wetsuit on a jet ski",
+      " a picture of a body of water with the words boats on it",
+      " a person in the water with the words if they rolled",
+      " a person in the water with a paddle",
+      " a person in the water with a scooter"
+    ],
+    "sound_mplug": "scuba divers rolling backwards off a boat while playing an instrument showcases exuberant laughter, splashing water, and cheery melodies blending with the gentle waves.",
+    "raw": "WHY SCUBA DIVERS ROLL BACKWARDS OFF BOAT #shorts"
+  },
+  "id": {
+    "folder": "video_folder",
+    "mplug": "mplug_caption",
+    "polish_mplug": "polish_mplug_caption",
+    "ofa": [
+      "ofa_caption_0",
+      "ofa_caption_1",
+      "ofa_caption_2",
+      "ofa_caption_3",
+      "ofa_caption_4",
+      "ofa_caption_5",
+      "ofa_caption_6",
+      "ofa_caption_7"
+    ],
+    "sound_mplug": "sound_mplug_caption",
+    "raw": "raw_caption#hashtags"
+  },
+  ...
+}
+```
+
+### Depth and Thermal (Infrared)
+
+We are uploading data to [Hugging Face](https://huggingface.co/datasets/LanguageBind/VIDAL-Depth-Thermal), but based on a conservative estimate, it's approximately **20T**. Please be patient as we work on it.

RAG/Knowledge_Database/languagebind_main/DATASET_LICENSE

@@ -0,0 +1,400 @@
+
+Attribution-NonCommercial 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+	wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More_considerations
+     for the public: 
+	wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution-NonCommercial 4.0 International Public
+License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial 4.0 International Public License ("Public
+License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You
+such rights in consideration of benefits the Licensor receives from
+making the Licensed Material available under these terms and
+conditions.
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+  d. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  e. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  f. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  g. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  h. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  i. NonCommercial means not primarily intended for or directed towards
+     commercial advantage or monetary compensation. For purposes of
+     this Public License, the exchange of the Licensed Material for
+     other material subject to Copyright and Similar Rights by digital
+     file-sharing or similar means is NonCommercial provided there is
+     no payment of monetary compensation in connection with the
+     exchange.
+
+  j. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  k. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  l. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part, for NonCommercial purposes only; and
+
+            b. produce, reproduce, and Share Adapted Material for
+               NonCommercial purposes only.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties, including when
+          the Licensed Material is used other than for NonCommercial
+          purposes.
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+       4. If You Share Adapted Material You produce, the Adapter's
+          License You apply must not prevent recipients of the Adapted
+          Material from complying with this Public License.
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database for NonCommercial purposes
+     only;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.

RAG/Knowledge_Database/languagebind_main/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 PKU-YUAN's Group (袁粒课题组-北大信工)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

RAG/Knowledge_Database/languagebind_main/README.md

@@ -0,0 +1,422 @@
+<p align="center">
+    <img src="assets/logo.jpg" width="350" style="margin-bottom: 0.2;"/><img src="assets/sota.jpg" width="450" style="margin-bottom: 0.2;"/>
+<p>
+<h2 align="center"> <a href="https://arxiv.org/pdf/2310.01852.pdf">【ICLR 2024 🔥】LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment</a></h2>
+<h5 align="center"> If you like our project, please give us a star ⭐ on GitHub for latest update.  </h2>
+
+ 
+<h5 align="center">
+     
+[![hf_space](https://img.shields.io/badge/🤗-Open%20In%20Spaces-blue.svg)](https://huggingface.co/spaces/LanguageBind/LanguageBind)
+[![Dataset meta](https://img.shields.io/badge/%F0%9F%A4%97%20Dataset-VIDAL-blue)](https://huggingface.co/datasets/LanguageBind/VIDAL-Depth-Thermal) 
+[![arXiv](https://img.shields.io/badge/Arxiv-2310.01852-b31b1b.svg?logo=arXiv)](https://arxiv.org/abs/2310.01852)
+[![wechat](https://img.shields.io/badge/量子位%20-black)](https://mp.weixin.qq.com/s/EFqLv_Euf5VU024zOtzkkg)
+[![jiqizhixin](https://img.shields.io/badge/机器之心%20-black)](https://mp.weixin.qq.com/s/E5Tazm_vz1CADMwV0tdhnw)
+[![zhihu](https://img.shields.io/badge/知乎-0084FF)](https://zhuanlan.zhihu.com/p/660567767)
+[![License](https://img.shields.io/badge/Code%20License-MIT-yellow)](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/LICENSE)
+[![Data License](https://img.shields.io/badge/Dataset%20license-CC--BY--NC%204.0-orange)](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/DATASET_LICENSE)
+[![Hits](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fgithub.com%2FPKU-YuanGroup%2FLanguageBind&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=Visitor&edge_flat=false)](https://hits.seeyoufarm.com) 
+[![GitHub issues](https://img.shields.io/github/issues/PKU-YuanGroup/LanguageBind?color=critical&label=Issues)](https://github.com/PKU-YuanGroup/LanguageBind/issues?q=is%3Aopen+is%3Aissue)
+[![GitHub closed issues](https://img.shields.io/github/issues-closed/PKU-YuanGroup/LanguageBind?color=success&label=Issues)](https://github.com/PKU-YuanGroup/LanguageBind/issues?q=is%3Aissue+is%3Aclosed)  <br>
+
+</h5>
+
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-audio-classification-on-audioset)](https://paperswithcode.com/sota/zero-shot-audio-classification-on-audioset?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-audio-classification-on-vgg-sound)](https://paperswithcode.com/sota/zero-shot-audio-classification-on-vgg-sound?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-text-to-audio-retrieval-on-clotho)](https://paperswithcode.com/sota/zero-shot-text-to-audio-retrieval-on-clotho?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-scene-classification-unified)](https://paperswithcode.com/sota/zero-shot-scene-classification-unified?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-classification-unified-classes-on)](https://paperswithcode.com/sota/zero-shot-classification-unified-classes-on?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-video-retrieval-on-msvd)](https://paperswithcode.com/sota/zero-shot-video-retrieval-on-msvd?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-environment-sound-classification-on-1)](https://paperswithcode.com/sota/zero-shot-environment-sound-classification-on-1?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-text-to-audio-retrieval-on)](https://paperswithcode.com/sota/zero-shot-text-to-audio-retrieval-on?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-video-retrieval-on-activitynet)](https://paperswithcode.com/sota/zero-shot-video-retrieval-on-activitynet?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-video-retrieval-on-msr-vtt)](https://paperswithcode.com/sota/zero-shot-video-retrieval-on-msr-vtt?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-video-retrieval-on-didemo)](https://paperswithcode.com/sota/zero-shot-video-retrieval-on-didemo?p=languagebind-extending-video-language) <br>
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/languagebind-extending-video-language/zero-shot-action-recognition-on-kinetics)](https://paperswithcode.com/sota/zero-shot-action-recognition-on-kinetics?p=languagebind-extending-video-language) 
+
+<details open><summary>💡 I also have other vision-language projects that may interest you ✨. </summary><p>
+<!--  may -->
+
+> [**Video-LLaVA: Learning United Visual Representation by Alignment Before Projection**](https://arxiv.org/abs/2311.10122) <br>
+> Bin Lin, Yang Ye, Bin Zhu, Jiaxi Cui, Munan Ning, Peng Jin, Li Yuan <br>
+[![github](https://img.shields.io/badge/-Github-black?logo=github)](https://github.com/PKU-YuanGroup/Video-LLaVA)  [![github](https://img.shields.io/github/stars/PKU-YuanGroup/Video-LLaVA.svg?style=social)](https://github.com/PKU-YuanGroup/Video-LLaVA) [![arXiv](https://img.shields.io/badge/Arxiv-2311.10122-b31b1b.svg?logo=arXiv)](https://arxiv.org/abs/2311.10122) <br>
+
+> [**MoE-LLaVA: Mixture of Experts for Large Vision-Language Models**](https://github.com/PKU-YuanGroup/MoE-LLaVA/blob/main/MoE-LLaVA.pdf) <br>
+> Bin Lin, Zhenyu Tang, Yang Ye, Jiaxi Cui, Bin Zhu, Peng Jin, Junwu Zhang, Munan Ning, Li Yuan <br>
+[![github](https://img.shields.io/badge/-Github-black?logo=github)](https://github.com/PKU-YuanGroup/MoE-LLaVA)  [![github](https://img.shields.io/github/stars/PKU-YuanGroup/MoE-LLaVA.svg?style=social)](https://github.com/PKU-YuanGroup/MoE-LLaVA) [![arXiv](https://img.shields.io/badge/Arxiv-2401.15947-b31b1b.svg?logo=arXiv)](https://arxiv.org/abs/2401.15947) <br>
+
+> [**Video-Bench: A Comprehensive Benchmark and Toolkit for Evaluating Video-based Large Language Models**](https://arxiv.org/abs/2311.08046) <br>
+> Munan Ning, Bin Zhu, Yujia Xie, Bin Lin, Jiaxi Cui, Lu Yuan, Dongdong Chen, Li Yuan <br>
+[![github](https://img.shields.io/badge/-Github-black?logo=github)](https://github.com/PKU-YuanGroup/Video-Bench) [![github](https://img.shields.io/github/stars/PKU-YuanGroup/Video-Bench.svg?style=social)](https://github.com/PKU-YuanGroup/Video-Bench) [![arXiv](https://img.shields.io/badge/Arxiv-2311.16103-b31b1b.svg?logo=arXiv)](https://arxiv.org/abs/2311.16103) <br>
+
+
+</p></details>
+
+## 📰 News
+* **[2024.01.27]**  👀👀👀 Our [MoE-LLaVA](https://github.com/PKU-YuanGroup/MoE-LLaVA) is released! A sparse model with 3B parameters outperformed the dense model with 7B parameters.
+* **[2024.01.16]**  🔥🔥🔥 Our LanguageBind has been accepted at ICLR 2024! We earn the score of 6(3)8(6)6(6)6(6) [here](https://openreview.net/forum?id=QmZKc7UZCy&noteId=OgsxQxAleA).
+* **[2023.12.15]**  💪💪💪 We expand the 💥💥💥 VIDAL dataset and now have **10M video-text data**. We launch **LanguageBind_Video 1.5**, checking our [model zoo](#-model-zoo). 
+* **[2023.12.10]**  We expand the 💥💥💥 VIDAL dataset and now have **10M depth and 10M thermal data**. We are in the process of uploading thermal and depth data on [Hugging Face](https://huggingface.co/datasets/LanguageBind/VIDAL-Depth-Thermal) and expect the whole process to last 1-2 months.
+* **[2023.11.27]**  🔥🔥🔥 We have updated our [paper](https://arxiv.org/abs/2310.01852) with emergency zero-shot results., checking our ✨ [results](#emergency-results).
+* **[2023.11.26]**  💥💥💥 We have open-sourced all textual sources and corresponding YouTube IDs [here](DATASETS.md).
+* **[2023.11.26]**  📣📣📣 We have open-sourced fully fine-tuned **Video & Audio**, achieving improved performance once again, checking our [model zoo](#-model-zoo). 
+* **[2023.11.22]**  We are about to release a fully fine-tuned version, and the **HUGE** version is currently undergoing training.
+* **[2023.11.21]**  💥 We are releasing sample data in [DATASETS.md](DATASETS.md) so that individuals who are interested can further modify the code to train it on their own data.
+* **[2023.11.20]**  🚀🚀🚀 [Video-LLaVA](https://github.com/PKU-YuanGroup/Video-LLaVA) builds a large visual-language model to achieve 🎉SOTA performances based on LanguageBind encoders.
+* **[2023.10.23]**  🎶 LanguageBind-Audio achieves 🎉🎉🎉**state-of-the-art (SOTA) performance on 5 datasets**, checking our ✨ [results](#multiple-modalities)!
+* **[2023.10.14]**  😱 Released a stronger LanguageBind-Video, checking our ✨ [results](#video-language)! The video checkpoint **have updated** on Huggingface Model Hub!
+* **[2023.10.10]**  We provide sample data, which can be found in [assets](assets), and [emergency zero-shot usage](#emergency-zero-shot) is described. 
+* **[2023.10.07]**  The checkpoints are available on 🤗 [Huggingface Model](https://huggingface.co/LanguageBind).
+* **[2023.10.04]**  Code and [demo](https://huggingface.co/spaces/LanguageBind/LanguageBind) are available now! Welcome to **watch** 👀 this repository for the latest updates.
+
+## 😮 Highlights
+
+### 💡 High performance, but NO intermediate modality required
+LanguageBind is a **language-centric** multimodal pretraining approach, **taking the language as the bind across different modalities** because the language modality is well-explored and contains rich semantics. 
+* The following first figure shows the architecture of LanguageBind. LanguageBind can be easily extended to segmentation, detection tasks, and potentially to unlimited modalities. 
+
+### ⚡️ A multimodal, fully aligned and voluminous dataset
+We propose **VIDAL-10M**, **10 Million data** with **V**ideo, **I**nfrared, **D**epth, **A**udio and their corresponding **L**anguage, which greatly expands the data beyond visual modalities.
+* The second figure shows our proposed VIDAL-10M dataset, which includes five modalities: video, infrared, depth, audio, and language.
+
+### 🔥 Multi-view enhanced description for training
+We make multi-view enhancements to language. We produce multi-view description that combines **meta-data**, **spatial**, and **temporal** to greatly enhance the semantic information of the language. In addition we further **enhance the language with ChatGPT** to create a good semantic space for each modality aligned language.
+
+<p align="center">
+<img src="assets/languagebind.jpg" width=100%>
+</p>
+<p align="center">
+<img src="assets/iclr_dataset_sample.jpg" width=99%>
+</p>
+
+## 🤗 Demo
+
+* **Local demo.** Highly recommend trying out our web demo, which incorporates all features currently supported by LanguageBind.
+```bash
+python gradio_app.py
+```
+
+* **Online demo.** We provide the [online demo](https://huggingface.co/spaces/LanguageBind/LanguageBind) in Huggingface Spaces. In this demo, you can calculate the similarity of modalities to language, such as audio-to-language, video-to-language, and depth-to-image.
+<p align="center">
+<img src="assets/demo.png" width=100%>
+</p>
+
+
+
+## 🚀 Main Results
+
+### Video-Language
+LanguageBind achieves **state-of-the-art (SOTA) performance on four datasets**, * donates the results of full tuning.
+<p align="left">
+<img src="assets/result1.jpg" width=80%>
+</p>
+
+### Multiple Modalities
+Video-Language, Infrared-Language, Depth-Language, and Audio-Language zero-shot classification, * donates the results of full tuning.
+<p align="left">
+<img src="assets/res1.jpg" width=80%>
+</p>
+We report text-to-audio results for retrieval, * donates the results of full tuning.
+<p align="left">
+<img src="assets/res2.jpg" width=35%>
+</p>
+
+### Emergency results
+<p align="left">
+<img src="assets/emergency.jpg" width=60%>
+</p>
+
+## 🛠️ Requirements and Installation
+* Python >= 3.8
+* Pytorch >= 1.13.1
+* CUDA Version >= 11.6
+* Install required packages:
+```bash
+git clone https://github.com/PKU-YuanGroup/LanguageBind
+cd LanguageBind
+pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
+pip install -r requirements.txt
+```
+
+## 🐳 Model Zoo
+
+The names in the table represent different encoder models. For example, `LanguageBind/LanguageBind_Video_FT` represents the fully fine-tuned version, while `LanguageBind/LanguageBind_Video` represents the LoRA-tuned version. 
+
+You can freely replace them in the recommended [API usage](#-api). We recommend using the fully fine-tuned version, as it offers stronger performance.
+
+<div align="center">
+<table border="1" width="100%">
+    <tr align="center">
+        <th>Modality</th><th>LoRA tuning</th><th>Fine-tuning</th>
+    </tr>
+    <tr align="center">
+        <td>Video</td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Video">LanguageBind_Video</a></td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Video_FT">LanguageBind_Video_FT</a></td>
+    </tr>
+    <tr align="center">
+        <td>Audio</td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Audio">LanguageBind_Audio</a></td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Audio_FT">LanguageBind_Audio_FT</a></td>
+    </tr>
+    <tr align="center">
+        <td>Depth</td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Depth">LanguageBind_Depth</a></td><td>-</td>
+    </tr>
+    <tr align="center">
+        <td>Thermal</td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Thermal">LanguageBind_Thermal</a></td><td>-</td>
+    </tr>
+</table>
+</div>
+
+
+<div align="center">
+<table border="1" width="100%">
+    <tr align="center">
+        <th>Version</th><th>Tuning</th><th>Model size</th><th>Num_frames</th><th>HF Link</th><th>MSR-VTT</th><th>DiDeMo</th><th>ActivityNet</th><th>MSVD</th>
+    </tr>
+    <tr align="center">
+        <td>LanguageBind_Video</td><td>LoRA</td><td>Large</td><td>8</td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Video">Link</a></td><td>42.6</td><td>37.8</td><td>35.1</td><td>52.2</td>
+    </tr>
+    <tr align="center">
+        <td>LanguageBind_Video_FT</td><td>Full-tuning</td><td>Large</td><td>8</td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Video_FT">Link</a></td><td>42.7</td><td>38.1</td><td>36.9</td><td>53.5</td>
+    </tr>
+    <tr align="center">
+        <td>LanguageBind_Video_V1.5_FT</td><td>Full-tuning</td><td>Large</td><td>8</td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Video_V1.5_FT">Link</a></td><td>42.8</td><td>39.7</td><td>38.4</td><td>54.1</td>
+    </tr>
+    <tr align="center">
+        <td>LanguageBind_Video_V1.5_FT</td><td>Full-tuning</td><td>Large</td><td>12</td><td>Coming soon</td>
+    </tr>
+    <tr align="center">
+        <td>LanguageBind_Video_Huge_V1.5_FT</td><td>Full-tuning</td><td>Huge</td><td>8</td><td><a href="https://huggingface.co/LanguageBind/LanguageBind_Video_Huge_V1.5_FT">Link</a></td><td>44.8</td><td>39.9</td><td>41.0</td><td>53.7</td>
+    </tr>
+    <tr align="center">
+        <td>LanguageBind_Video_Huge_V1.5_FT</td><td>Full-tuning</td><td>Huge</td><td>12</td><td>Coming soon</td>
+    </tr>
+</table>
+</div>
+
+## 🤖 API
+**We open source all modalities preprocessing code.** If you want to load the model (e.g. ```LanguageBind/LanguageBind_Thermal```) from the model hub on Huggingface or on local, you can use the following code snippets!
+### Inference for Multi-modal Binding 
+We have provided some sample datasets in [assets](assets) to quickly see how languagebind works.
+```python
+import torch
+from languagebind import LanguageBind, to_device, transform_dict, LanguageBindImageTokenizer
+
+if __name__ == '__main__':
+    device = 'cuda:0'
+    device = torch.device(device)
+    clip_type = {
+        'video': 'LanguageBind_Video_FT',  # also LanguageBind_Video
+        'audio': 'LanguageBind_Audio_FT',  # also LanguageBind_Audio
+        'thermal': 'LanguageBind_Thermal',
+        'image': 'LanguageBind_Image',
+        'depth': 'LanguageBind_Depth',
+    }
+
+    model = LanguageBind(clip_type=clip_type, cache_dir='./cache_dir')
+    model = model.to(device)
+    model.eval()
+    pretrained_ckpt = f'lb203/LanguageBind_Image'
+    tokenizer = LanguageBindImageTokenizer.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir/tokenizer_cache_dir')
+    modality_transform = {c: transform_dict[c](model.modality_config[c]) for c in clip_type.keys()}
+
+    image = ['assets/image/0.jpg', 'assets/image/1.jpg']
+    audio = ['assets/audio/0.wav', 'assets/audio/1.wav']
+    video = ['assets/video/0.mp4', 'assets/video/1.mp4']
+    depth = ['assets/depth/0.png', 'assets/depth/1.png']
+    thermal = ['assets/thermal/0.jpg', 'assets/thermal/1.jpg']
+    language = ["Training a parakeet to climb up a ladder.", 'A lion climbing a tree to catch a monkey.']
+
+    inputs = {
+        'image': to_device(modality_transform['image'](image), device),
+        'video': to_device(modality_transform['video'](video), device),
+        'audio': to_device(modality_transform['audio'](audio), device),
+        'depth': to_device(modality_transform['depth'](depth), device),
+        'thermal': to_device(modality_transform['thermal'](thermal), device),
+    }
+    inputs['language'] = to_device(tokenizer(language, max_length=77, padding='max_length',
+                                             truncation=True, return_tensors='pt'), device)
+
+    with torch.no_grad():
+        embeddings = model(inputs)
+
+    print("Video x Text: \n",
+          torch.softmax(embeddings['video'] @ embeddings['language'].T, dim=-1).detach().cpu().numpy())
+    print("Image x Text: \n",
+          torch.softmax(embeddings['image'] @ embeddings['language'].T, dim=-1).detach().cpu().numpy())
+    print("Depth x Text: \n",
+          torch.softmax(embeddings['depth'] @ embeddings['language'].T, dim=-1).detach().cpu().numpy())
+    print("Audio x Text: \n",
+          torch.softmax(embeddings['audio'] @ embeddings['language'].T, dim=-1).detach().cpu().numpy())
+    print("Thermal x Text: \n",
+          torch.softmax(embeddings['thermal'] @ embeddings['language'].T, dim=-1).detach().cpu().numpy())
+```
+Then returns the following result.
+```bash
+Video x Text: 
+ [[9.9989331e-01 1.0667283e-04]
+ [1.3255903e-03 9.9867439e-01]]
+Image x Text: 
+ [[9.9990666e-01 9.3292067e-05]
+ [4.6132666e-08 1.0000000e+00]]
+Depth x Text: 
+ [[0.9954276  0.00457235]
+ [0.12042473 0.8795753 ]]
+Audio x Text: 
+ [[0.97634876 0.02365119]
+ [0.02917843 0.97082156]]
+Thermal x Text: 
+ [[0.9482511  0.0517489 ]
+ [0.48746133 0.5125386 ]]
+```
+### Emergency zero-shot
+Since languagebind binds each modality together, we also found the **emergency zero-shot**. It's very simple to use.
+```python
+print("Video x Audio: \n", torch.softmax(embeddings['video'] @ embeddings['audio'].T, dim=-1).detach().cpu().numpy())
+print("Image x Depth: \n", torch.softmax(embeddings['image'] @ embeddings['depth'].T, dim=-1).detach().cpu().numpy())
+print("Image x Thermal: \n", torch.softmax(embeddings['image'] @ embeddings['thermal'].T, dim=-1).detach().cpu().numpy())
+```
+Then, you will get:
+```
+Video x Audio: 
+ [[1.0000000e+00 0.0000000e+00]
+ [3.1150486e-32 1.0000000e+00]]
+Image x Depth: 
+ [[1. 0.]
+ [0. 1.]]
+Image x Thermal: 
+ [[1. 0.]
+ [0. 1.]]
+ ```
+
+### Different branches for X-Language task
+Additionally, LanguageBind can be **disassembled into different branches** to handle different tasks. Note that we do not train Image, which just initialize from OpenCLIP.
+#### Thermal
+```python
+import torch
+from languagebind import LanguageBindThermal, LanguageBindThermalTokenizer, LanguageBindThermalProcessor
+
+pretrained_ckpt = 'LanguageBind/LanguageBind_Thermal'
+model = LanguageBindThermal.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+tokenizer = LanguageBindThermalTokenizer.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+thermal_process = LanguageBindThermalProcessor(model.config, tokenizer)
+
+model.eval()
+data = thermal_process([r"your/thermal.jpg"], ['your text'], return_tensors='pt')
+with torch.no_grad():
+    out = model(**data)
+
+print(out.text_embeds @ out.image_embeds.T)
+```
+
+#### Depth
+```python
+import torch
+from languagebind import LanguageBindDepth, LanguageBindDepthTokenizer, LanguageBindDepthProcessor
+
+pretrained_ckpt = 'LanguageBind/LanguageBind_Depth'
+model = LanguageBindDepth.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+tokenizer = LanguageBindDepthTokenizer.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+depth_process = LanguageBindDepthProcessor(model.config, tokenizer)
+
+model.eval()
+data = depth_process([r"your/depth.png"], ['your text.'], return_tensors='pt')
+with torch.no_grad():
+    out = model(**data)
+
+print(out.text_embeds @ out.image_embeds.T)
+```
+
+#### Video
+```python
+import torch
+from languagebind import LanguageBindVideo, LanguageBindVideoTokenizer, LanguageBindVideoProcessor
+
+pretrained_ckpt = 'LanguageBind/LanguageBind_Video_FT'  # also 'LanguageBind/LanguageBind_Video'
+model = LanguageBindVideo.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+tokenizer = LanguageBindVideoTokenizer.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+video_process = LanguageBindVideoProcessor(model.config, tokenizer)
+
+model.eval()
+data = video_process(["your/video.mp4"], ['your text.'], return_tensors='pt')
+with torch.no_grad():
+    out = model(**data)
+
+print(out.text_embeds @ out.image_embeds.T)
+```
+
+#### Audio
+```python
+import torch
+from languagebind import LanguageBindAudio, LanguageBindAudioTokenizer, LanguageBindAudioProcessor
+
+pretrained_ckpt = 'LanguageBind/LanguageBind_Audio_FT'  # also 'LanguageBind/LanguageBind_Audio'
+model = LanguageBindAudio.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+tokenizer = LanguageBindAudioTokenizer.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+audio_process = LanguageBindAudioProcessor(model.config, tokenizer)
+
+model.eval()
+data = audio_process([r"your/audio.wav"], ['your audio.'], return_tensors='pt')
+with torch.no_grad():
+    out = model(**data)
+
+print(out.text_embeds @ out.image_embeds.T)
+```
+
+#### Image
+Note that our image encoder is the same as OpenCLIP. **Not** as fine-tuned as other modalities.
+```python
+import torch
+from languagebind import LanguageBindImage,  LanguageBindImageTokenizer,  LanguageBindImageProcessor
+
+pretrained_ckpt = 'LanguageBind/LanguageBind_Image'
+model = LanguageBindImage.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+tokenizer = LanguageBindImageTokenizer.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir')
+image_process = LanguageBindImageProcessor(model.config, tokenizer)
+
+model.eval()
+data = image_process([r"your/image.jpg"], ['your text.'], return_tensors='pt')
+with torch.no_grad():
+    out = model(**data)
+
+print(out.text_embeds @ out.image_embeds.T)
+```
+
+## 💥 VIDAL-10M
+The datasets is in [DATASETS.md](DATASETS.md).
+
+## 🗝️ Training & Validating
+The training & validating instruction is in [TRAIN_AND_VALIDATE.md](TRAIN_AND_VALIDATE.md).
+
+## 👍 Acknowledgement
+* [OpenCLIP](https://github.com/mlfoundations/open_clip) An open source pretraining framework.
+* [CLIP4Clip](https://github.com/ArrowLuo/CLIP4Clip) An open source Video-Text retrieval framework.
+* [sRGB-TIR](https://github.com/rpmsnu/sRGB-TIR) An open source framework to generate infrared (thermal) images.
+* [GLPN](https://github.com/vinvino02/GLPDepth) An open source framework to generate depth images.
+
+## 🔒 License
+* The majority of this project is released under the MIT license as found in the [LICENSE](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/LICENSE) file.
+* The dataset of this project is released under the CC-BY-NC 4.0 license as found in the [DATASET_LICENSE](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/DATASET_LICENSE) file. 
+
+## ✏️ Citation
+If you find our paper and code useful in your research, please consider giving a star :star: and citation :pencil:.
+
+```BibTeX
+@misc{zhu2023languagebind,
+      title={LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment}, 
+      author={Bin Zhu and Bin Lin and Munan Ning and Yang Yan and Jiaxi Cui and Wang HongFa and Yatian Pang and Wenhao Jiang and Junwu Zhang and Zongwei Li and Cai Wan Zhang and Zhifeng Li and Wei Liu and Li Yuan},
+      year={2023},
+      eprint={2310.01852},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV}
+}
+```
+
+
+## ✨ Star History
+
+[![Star History](https://api.star-history.com/svg?repos=PKU-YuanGroup/LanguageBind&type=Date)](https://star-history.com/#PKU-YuanGroup/LanguageBind&Date)
+
+
+## 🤝 Contributors
+
+<a href="https://github.com/PKU-YuanGroup/LanguageBind/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=PKU-YuanGroup/LanguageBind" />
+</a>

RAG/Knowledge_Database/languagebind_main/TRAIN_AND_VALIDATE.md

@@ -0,0 +1,214 @@
+We provide the **off-the-shelf** scripts in the [scripts folder](scripts).
+
+## Training LanguageBind 
+
+
+<div align="center">
+<table border="1" width="100%">
+    <tr align="center">
+        <th>Cache of pretrained weight</th><th>Baidu Yun</th><th>Google Cloud</th><th>Peking University Yun</th>
+    </tr>
+    <tr align="center">
+        <td>Large</td><td><a href="https://pan.baidu.com/s/1co46bkuUJXr8ePPKp1WWgA?pwd=ofm6">Link</a></td><td><a href="https://drive.google.com/drive/folders/1VQYZlqfKmCMuHffypf5F96odyMCEI87H?usp=drive_link">Link</a></td><td><a href="https://disk.pku.edu.cn:443/link/9CA764E6307790B01D2D4F7E314E8E43">Link</a></td>
+    </tr>
+    <tr align="center">
+        <td>Huge</td><td><a href="https://pan.baidu.com/s/1QLpyXEYunoXS-oqGsvzKKA?pwd=vgo2">Link</a></td><td>-</td><td><a href="https://disk.pku.edu.cn:443/link/720A77A7DB9EFD167C5AC8E3FC4B6068">Link</a></td>
+    </tr>
+</table>
+</div>
+
+
+For example, to **train** LanguageBind on **Depth-Language** with 8 GPUs (1 nodes x 8 GPUs).
+* First download the cache of pretrained weight above. and specify `CACHE_DIR=path/to/LanguageBind`.
+* The second step is to develop a path to `ANNOTATION` and `DATA` [here](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/data/base_datasets.py#L37) according to the [dataset preparation](https://github.com/PKU-YuanGroup/LanguageBind#-vidal-10m).
+* Then you can run
+
+```bash
+CACHE_DIR="/path/to/LanguageBind"
+ANNOTATION="path/to/data"
+cd /path/to/LanguageBind
+TORCH_DISTRIBUTED_DEBUG=DETAIL HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 torchrun --nnodes=1 --nproc_per_node 8 \
+    -m main  \
+    --train-data ${ANNOTATION} \
+    --train-num-samples 3020000 \
+    --clip-type "dl" --max-depth 10 \
+    --do_train \
+    --lock-text --lock-image --text-type "polish_mplug" \
+    --init-temp 0.07 --learn-temp \
+    --model "ViT-L-14" --cache-dir ${CACHE_DIR} \
+    --convert_to_lora --lora_r 2 \
+    --lr 5e-4 --coef-lr 1e-3 \
+    --beta1 0.9 --beta2 0.98 --wd 0.2 --eps 1e-6 \
+    --num-frames 1 --force-patch-dropout 0.5 \
+    --epochs 1 --batch-size 128 --accum-freq 1 --warmup 200 \
+    --precision "amp" --workers 10 --video-decode-backend "imgs" \
+    --save-frequency 1 --log-every-n-steps 20 --report-to "tensorboard" --resume "latest" \
+    --do_eval \
+    --val_d_cls_data "NYUV2"
+```
+
+
+## Validating LanguageBind 
+
+For example, to **validate** LanguageBind on **Depth-Language** with 1 GPUs.
+* First specify ```RESUME```.
+* The second step is to prepare the [downstream dataset](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/TRAIN_AND_VALIDATE.md#downstream-datasets).
+* Then you can run
+
+```bash
+CACHE_DIR="/path/to/LanguageBind"
+RESUME="thermal_language.pt"
+ANNOTATION="path/to/data"
+cd /path/to/LanguageBind
+TORCH_DISTRIBUTED_DEBUG=DETAIL HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 torchrun --nproc_per_node 1 \
+    -m main  \
+    --train-data ${ANNOTATION} \
+    --train-num-samples 3020000 \
+    --clip-type "dl" --max-depth 10 \
+    --lock-text --lock-image --text-type "polish_mplug" \
+    --init-temp 0.07 --learn-temp \
+    --model "ViT-L-14" --cache-dir ${CACHE_DIR} \
+    --convert_to_lora --lora_r 2 \
+    --lr 5e-4 --coef-lr 1e-3 \
+    --beta1 0.9 --beta2 0.98 --wd 0.2 --eps 1e-6 \
+    --num-frames 1 --force-patch-dropout 0.5 \
+    --epochs 1 --batch-size 128 --accum-freq 1 --warmup 200 \
+    --precision "amp" --workers 10 --video-decode-backend "imgs" \
+    --save-frequency 1 --log-every-n-steps 20 --report-to "tensorboard" --resume ${RESUME} \
+    --do_eval \
+    --val_d_cls_data "NYUV2"
+```
+
+## Downstream datasets
+
+### Depth
+NYU V2 dataset is downloaded from [this repo](https://github.com/TUI-NICR/nicr-scene-analysis-datasets/tree/main/nicr_scene_analysis_datasets/datasets/nyuv2) and we reformat them to conform to the standard ImageNet format. We also provide data as follows. Change the ```data_root``` [here](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/data/build_datasets.py#L221).
+
+<div align="center">
+<table border="1" width="100%">
+    <tr align="center">
+        <th>Datasets</th><th>Baidu Yun</th><th>Google Cloud</th><th>Peking University Yun</th>
+    </tr>
+    <tr align="center">
+        <td>NYU</td><td><a href="https://pan.baidu.com/s/1AGOG8U3F7W8AvJiEmuzs-A?pwd=1dsg">Link</a></td><td><a href="https://drive.google.com/file/d/1CltzrTBLFqLxJzpztSIN-5ZosZpXQQ6u/view?usp=sharing">Link</a></td><td><a href="https://disk.pku.edu.cn:443/link/7D7B164DEA64059793D3C3E3A65C0F64">Link</a></td>
+    </tr>
+</table>
+</div>
+
+### Video
+Video datasets are downloaded from [this repo](https://github.com/jpthu17/HBI) and we show the folder structure. Change the ```data_root``` [here](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/data/build_datasets.py#L74).
+
+### Audio
+Audio datasets are downloaded from [this repo](https://github.com/OFA-Sys/ONE-PEACE/blob/main/datasets.md#audio) and Audioset from [here](https://github.com/qiuqiangkong/audioset_tagging_cnn#1-download-dataset).We reformat them to conform to the standard ImageNet format. Change the ```data_root``` [here1](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/data/build_datasets.py#L144) and [here2](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/data/build_datasets.py#L159).
+
+### Infrared (Thermal)
+We download LLVIP from [official website](https://bupt-ai-cz.github.io/LLVIP/), and FLIR from [here](https://www.flir.com/oem/adas/adas-dataset-form/). We reformat them to conform to the standard ImageNet format. Change the ```data_root``` [here](https://github.com/PKU-YuanGroup/LanguageBind/blob/main/data/build_datasets.py#L233). We also provide the processed data as follows.
+
+<div align="center">
+<table border="1" width="100%">
+    <tr align="center">
+        <th>Datasets</th><th>Baidu Yun</th><th>Google Cloud</th><th>Peking University Yun</th>
+    </tr>
+    <tr align="center">
+        <td>LLVIP</td><td><a href="https://pan.baidu.com/s/15HPVr016F7eO9005NDRJTg?pwd=46fh">Link</a></td><td><a href="https://drive.google.com/file/d/1RfKNR8q6dHiAHB4OlYecnkUSx-ghLuEO/view?usp=drive_link">Link</a></td><td><a href="https://disk.pku.edu.cn:443/link/30D592EA37AC7C411264801A74994376">Link</a></td>
+    </tr>
+    <tr align="center">
+        <td>FLIR V1</td><td><a href="https://pan.baidu.com/s/1ZDSo5VPxJ4SA7wS_rNk0uQ?pwd=l491">Link</a></td><td><a href="https://drive.google.com/file/d/1CezCLJ4GUfPMFimitPfK40OV2j2Kr8t8/view?usp=drive_link">Link</a></td><td><a href="https://disk.pku.edu.cn:443/link/AD89D6ADE2CAC2407B00650870CBBDEC">Link</a></td>
+    </tr>
+    <tr align="center">
+        <td>FLIR V2</td><td><a href="https://pan.baidu.com/s/16xdr2aQkHo3zJ4KbaTmO3Q?pwd=tj9f">Link</a></td><td><a href="https://drive.google.com/file/d/1Z2ThG5QH-9biFI2-Z8k2fBKSA6Nrees6/view?usp=drive_link">Link</a></td><td><a href="https://disk.pku.edu.cn:443/link/E06C010970B0ED51926700D2F7A21EA8">Link</a></td>
+    </tr>
+</table>
+</div>
+
+### Folder structure
+```bash
+downstream_datasets
+├── Audio
+│   ├── audiocaps
+│   │   └── audio
+│   │       ├── test
+│   │       ├── train
+│   │       └── val
+│   ├── audioset
+│   │   ├── balanced_train_segments
+│   │   ├── eval_segments
+│   │   └── unbalanced_train_segments
+│   │       ├── unbalanced_train_segments_part00
+│   │       ├── unbalanced_train_segments_part01
+│   │       ├── ...
+│   │       └── unbalanced_train_segments_part40
+│   ├── clotho
+│   │   ├── CLOTHO_retrieval_dataset
+│   │   └── evaluation
+│   ├── esc50
+│   │   └── test
+│   │       ├── airplane
+│   │       ├── breathing
+│   │       ├── ...
+│   │       └── wind
+├── laionaudio
+│   │   ├── audios
+│   │   ├── freesound_no_overlap
+│   │   └── jsons
+├── vggsound
+│       └── test
+│           ├── air\ conditioning\ noise
+│           ├── air\ horn
+│           ├── ...
+│           └── zebra\ braying
+├── Depth
+│   ├── nyuv2
+│   │   ├── data
+│   │   │   └── val
+│   │   │       ├── bathroom
+│   │   │       ├── bedroom
+│   │   │       ├── bookstore
+│   │   │       ├── classroom
+│   │   │       ├── dining_room
+│   │   │       ├── home_office
+│   │   │       ├── kitchen
+│   │   │       ├── living_room
+│   │   │       ├── office
+│   │   │       └── others
+├── Thermal
+│   ├── flirv1
+│   │   └── val
+│   │       ├── bicycle
+│   │       ├── car
+│   │       ├── dog
+│   │       └── person
+│   ├── flirv2
+│   │   └── val
+│   │       ├── bike
+│   │       ├── bus
+│   │       ├── car
+│   │       ├── hydrant
+│   │       ├── light
+│   │       ├── motor
+│   │       ├── other\ vehicle
+│   │       ├── person
+│   │       ├── sign
+│   │       ├── skateboard
+│   │       ├── stroller
+│   │       └── truck
+│   ├── llvip
+│   │   ├── train
+│   │   │   ├── background
+│   │   │   └── person
+│   │   └── val
+│   │       ├── background
+│   │       └── person
+└── VideoTextRetrieval
+    ├── vtRetdata
+    │   ├── ActivityNet
+    │   │   └── Videos
+    │   │       └── Activity_Videos
+    │   ├── Didemo
+    │   │   └── videos
+    │   ├── MSRVTT
+    │   │   └── MSRVTT_Videos
+    │   └── MSVD
+    │��      └── MSVD_Videos
+```
+

RAG/Knowledge_Database/languagebind_main/a_cls/class_labels_indices.csv

@@ -0,0 +1,528 @@
+index,mid,display_name
+0,/m/09x0r,"Speech"
+1,/m/05zppz,"Male speech, man speaking"
+2,/m/02zsn,"Female speech, woman speaking"
+3,/m/0ytgt,"Child speech, kid speaking"
+4,/m/01h8n0,"Conversation"
+5,/m/02qldy,"Narration, monologue"
+6,/m/0261r1,"Babbling"
+7,/m/0brhx,"Speech synthesizer"
+8,/m/07p6fty,"Shout"
+9,/m/07q4ntr,"Bellow"
+10,/m/07rwj3x,"Whoop"
+11,/m/07sr1lc,"Yell"
+12,/m/04gy_2,"Battle cry"
+13,/t/dd00135,"Children shouting"
+14,/m/03qc9zr,"Screaming"
+15,/m/02rtxlg,"Whispering"
+16,/m/01j3sz,"Laughter"
+17,/t/dd00001,"Baby laughter"
+18,/m/07r660_,"Giggle"
+19,/m/07s04w4,"Snicker"
+20,/m/07sq110,"Belly laugh"
+21,/m/07rgt08,"Chuckle, chortle"
+22,/m/0463cq4,"Crying, sobbing"
+23,/t/dd00002,"Baby cry, infant cry"
+24,/m/07qz6j3,"Whimper"
+25,/m/07qw_06,"Wail, moan"
+26,/m/07plz5l,"Sigh"
+27,/m/015lz1,"Singing"
+28,/m/0l14jd,"Choir"
+29,/m/01swy6,"Yodeling"
+30,/m/02bk07,"Chant"
+31,/m/01c194,"Mantra"
+32,/t/dd00003,"Male singing"
+33,/t/dd00004,"Female singing"
+34,/t/dd00005,"Child singing"
+35,/t/dd00006,"Synthetic singing"
+36,/m/06bxc,"Rapping"
+37,/m/02fxyj,"Humming"
+38,/m/07s2xch,"Groan"
+39,/m/07r4k75,"Grunt"
+40,/m/01w250,"Whistling"
+41,/m/0lyf6,"Breathing"
+42,/m/07mzm6,"Wheeze"
+43,/m/01d3sd,"Snoring"
+44,/m/07s0dtb,"Gasp"
+45,/m/07pyy8b,"Pant"
+46,/m/07q0yl5,"Snort"
+47,/m/01b_21,"Cough"
+48,/m/0dl9sf8,"Throat clearing"
+49,/m/01hsr_,"Sneeze"
+50,/m/07ppn3j,"Sniff"
+51,/m/06h7j,"Run"
+52,/m/07qv_x_,"Shuffle"
+53,/m/07pbtc8,"Walk, footsteps"
+54,/m/03cczk,"Chewing, mastication"
+55,/m/07pdhp0,"Biting"
+56,/m/0939n_,"Gargling"
+57,/m/01g90h,"Stomach rumble"
+58,/m/03q5_w,"Burping, eructation"
+59,/m/02p3nc,"Hiccup"
+60,/m/02_nn,"Fart"
+61,/m/0k65p,"Hands"
+62,/m/025_jnm,"Finger snapping"
+63,/m/0l15bq,"Clapping"
+64,/m/01jg02,"Heart sounds, heartbeat"
+65,/m/01jg1z,"Heart murmur"
+66,/m/053hz1,"Cheering"
+67,/m/028ght,"Applause"
+68,/m/07rkbfh,"Chatter"
+69,/m/03qtwd,"Crowd"
+70,/m/07qfr4h,"Hubbub, speech noise, speech babble"
+71,/t/dd00013,"Children playing"
+72,/m/0jbk,"Animal"
+73,/m/068hy,"Domestic animals, pets"
+74,/m/0bt9lr,"Dog"
+75,/m/05tny_,"Bark"
+76,/m/07r_k2n,"Yip"
+77,/m/07qf0zm,"Howl"
+78,/m/07rc7d9,"Bow-wow"
+79,/m/0ghcn6,"Growling"
+80,/t/dd00136,"Whimper (dog)"
+81,/m/01yrx,"Cat"
+82,/m/02yds9,"Purr"
+83,/m/07qrkrw,"Meow"
+84,/m/07rjwbb,"Hiss"
+85,/m/07r81j2,"Caterwaul"
+86,/m/0ch8v,"Livestock, farm animals, working animals"
+87,/m/03k3r,"Horse"
+88,/m/07rv9rh,"Clip-clop"
+89,/m/07q5rw0,"Neigh, whinny"
+90,/m/01xq0k1,"Cattle, bovinae"
+91,/m/07rpkh9,"Moo"
+92,/m/0239kh,"Cowbell"
+93,/m/068zj,"Pig"
+94,/t/dd00018,"Oink"
+95,/m/03fwl,"Goat"
+96,/m/07q0h5t,"Bleat"
+97,/m/07bgp,"Sheep"
+98,/m/025rv6n,"Fowl"
+99,/m/09b5t,"Chicken, rooster"
+100,/m/07st89h,"Cluck"
+101,/m/07qn5dc,"Crowing, cock-a-doodle-doo"
+102,/m/01rd7k,"Turkey"
+103,/m/07svc2k,"Gobble"
+104,/m/09ddx,"Duck"
+105,/m/07qdb04,"Quack"
+106,/m/0dbvp,"Goose"
+107,/m/07qwf61,"Honk"
+108,/m/01280g,"Wild animals"
+109,/m/0cdnk,"Roaring cats (lions, tigers)"
+110,/m/04cvmfc,"Roar"
+111,/m/015p6,"Bird"
+112,/m/020bb7,"Bird vocalization, bird call, bird song"
+113,/m/07pggtn,"Chirp, tweet"
+114,/m/07sx8x_,"Squawk"
+115,/m/0h0rv,"Pigeon, dove"
+116,/m/07r_25d,"Coo"
+117,/m/04s8yn,"Crow"
+118,/m/07r5c2p,"Caw"
+119,/m/09d5_,"Owl"
+120,/m/07r_80w,"Hoot"
+121,/m/05_wcq,"Bird flight, flapping wings"
+122,/m/01z5f,"Canidae, dogs, wolves"
+123,/m/06hps,"Rodents, rats, mice"
+124,/m/04rmv,"Mouse"
+125,/m/07r4gkf,"Patter"
+126,/m/03vt0,"Insect"
+127,/m/09xqv,"Cricket"
+128,/m/09f96,"Mosquito"
+129,/m/0h2mp,"Fly, housefly"
+130,/m/07pjwq1,"Buzz"
+131,/m/01h3n,"Bee, wasp, etc."
+132,/m/09ld4,"Frog"
+133,/m/07st88b,"Croak"
+134,/m/078jl,"Snake"
+135,/m/07qn4z3,"Rattle"
+136,/m/032n05,"Whale vocalization"
+137,/m/04rlf,"Music"
+138,/m/04szw,"Musical instrument"
+139,/m/0fx80y,"Plucked string instrument"
+140,/m/0342h,"Guitar"
+141,/m/02sgy,"Electric guitar"
+142,/m/018vs,"Bass guitar"
+143,/m/042v_gx,"Acoustic guitar"
+144,/m/06w87,"Steel guitar, slide guitar"
+145,/m/01glhc,"Tapping (guitar technique)"
+146,/m/07s0s5r,"Strum"
+147,/m/018j2,"Banjo"
+148,/m/0jtg0,"Sitar"
+149,/m/04rzd,"Mandolin"
+150,/m/01bns_,"Zither"
+151,/m/07xzm,"Ukulele"
+152,/m/05148p4,"Keyboard (musical)"
+153,/m/05r5c,"Piano"
+154,/m/01s0ps,"Electric piano"
+155,/m/013y1f,"Organ"
+156,/m/03xq_f,"Electronic organ"
+157,/m/03gvt,"Hammond organ"
+158,/m/0l14qv,"Synthesizer"
+159,/m/01v1d8,"Sampler"
+160,/m/03q5t,"Harpsichord"
+161,/m/0l14md,"Percussion"
+162,/m/02hnl,"Drum kit"
+163,/m/0cfdd,"Drum machine"
+164,/m/026t6,"Drum"
+165,/m/06rvn,"Snare drum"
+166,/m/03t3fj,"Rimshot"
+167,/m/02k_mr,"Drum roll"
+168,/m/0bm02,"Bass drum"
+169,/m/011k_j,"Timpani"
+170,/m/01p970,"Tabla"
+171,/m/01qbl,"Cymbal"
+172,/m/03qtq,"Hi-hat"
+173,/m/01sm1g,"Wood block"
+174,/m/07brj,"Tambourine"
+175,/m/05r5wn,"Rattle (instrument)"
+176,/m/0xzly,"Maraca"
+177,/m/0mbct,"Gong"
+178,/m/016622,"Tubular bells"
+179,/m/0j45pbj,"Mallet percussion"
+180,/m/0dwsp,"Marimba, xylophone"
+181,/m/0dwtp,"Glockenspiel"
+182,/m/0dwt5,"Vibraphone"
+183,/m/0l156b,"Steelpan"
+184,/m/05pd6,"Orchestra"
+185,/m/01kcd,"Brass instrument"
+186,/m/0319l,"French horn"
+187,/m/07gql,"Trumpet"
+188,/m/07c6l,"Trombone"
+189,/m/0l14_3,"Bowed string instrument"
+190,/m/02qmj0d,"String section"
+191,/m/07y_7,"Violin, fiddle"
+192,/m/0d8_n,"Pizzicato"
+193,/m/01xqw,"Cello"
+194,/m/02fsn,"Double bass"
+195,/m/085jw,"Wind instrument, woodwind instrument"
+196,/m/0l14j_,"Flute"
+197,/m/06ncr,"Saxophone"
+198,/m/01wy6,"Clarinet"
+199,/m/03m5k,"Harp"
+200,/m/0395lw,"Bell"
+201,/m/03w41f,"Church bell"
+202,/m/027m70_,"Jingle bell"
+203,/m/0gy1t2s,"Bicycle bell"
+204,/m/07n_g,"Tuning fork"
+205,/m/0f8s22,"Chime"
+206,/m/026fgl,"Wind chime"
+207,/m/0150b9,"Change ringing (campanology)"
+208,/m/03qjg,"Harmonica"
+209,/m/0mkg,"Accordion"
+210,/m/0192l,"Bagpipes"
+211,/m/02bxd,"Didgeridoo"
+212,/m/0l14l2,"Shofar"
+213,/m/07kc_,"Theremin"
+214,/m/0l14t7,"Singing bowl"
+215,/m/01hgjl,"Scratching (performance technique)"
+216,/m/064t9,"Pop music"
+217,/m/0glt670,"Hip hop music"
+218,/m/02cz_7,"Beatboxing"
+219,/m/06by7,"Rock music"
+220,/m/03lty,"Heavy metal"
+221,/m/05r6t,"Punk rock"
+222,/m/0dls3,"Grunge"
+223,/m/0dl5d,"Progressive rock"
+224,/m/07sbbz2,"Rock and roll"
+225,/m/05w3f,"Psychedelic rock"
+226,/m/06j6l,"Rhythm and blues"
+227,/m/0gywn,"Soul music"
+228,/m/06cqb,"Reggae"
+229,/m/01lyv,"Country"
+230,/m/015y_n,"Swing music"
+231,/m/0gg8l,"Bluegrass"
+232,/m/02x8m,"Funk"
+233,/m/02w4v,"Folk music"
+234,/m/06j64v,"Middle Eastern music"
+235,/m/03_d0,"Jazz"
+236,/m/026z9,"Disco"
+237,/m/0ggq0m,"Classical music"
+238,/m/05lls,"Opera"
+239,/m/02lkt,"Electronic music"
+240,/m/03mb9,"House music"
+241,/m/07gxw,"Techno"
+242,/m/07s72n,"Dubstep"
+243,/m/0283d,"Drum and bass"
+244,/m/0m0jc,"Electronica"
+245,/m/08cyft,"Electronic dance music"
+246,/m/0fd3y,"Ambient music"
+247,/m/07lnk,"Trance music"
+248,/m/0g293,"Music of Latin America"
+249,/m/0ln16,"Salsa music"
+250,/m/0326g,"Flamenco"
+251,/m/0155w,"Blues"
+252,/m/05fw6t,"Music for children"
+253,/m/02v2lh,"New-age music"
+254,/m/0y4f8,"Vocal music"
+255,/m/0z9c,"A capella"
+256,/m/0164x2,"Music of Africa"
+257,/m/0145m,"Afrobeat"
+258,/m/02mscn,"Christian music"
+259,/m/016cjb,"Gospel music"
+260,/m/028sqc,"Music of Asia"
+261,/m/015vgc,"Carnatic music"
+262,/m/0dq0md,"Music of Bollywood"
+263,/m/06rqw,"Ska"
+264,/m/02p0sh1,"Traditional music"
+265,/m/05rwpb,"Independent music"
+266,/m/074ft,"Song"
+267,/m/025td0t,"Background music"
+268,/m/02cjck,"Theme music"
+269,/m/03r5q_,"Jingle (music)"
+270,/m/0l14gg,"Soundtrack music"
+271,/m/07pkxdp,"Lullaby"
+272,/m/01z7dr,"Video game music"
+273,/m/0140xf,"Christmas music"
+274,/m/0ggx5q,"Dance music"
+275,/m/04wptg,"Wedding music"
+276,/t/dd00031,"Happy music"
+277,/t/dd00032,"Funny music"
+278,/t/dd00033,"Sad music"
+279,/t/dd00034,"Tender music"
+280,/t/dd00035,"Exciting music"
+281,/t/dd00036,"Angry music"
+282,/t/dd00037,"Scary music"
+283,/m/03m9d0z,"Wind"
+284,/m/09t49,"Rustling leaves"
+285,/t/dd00092,"Wind noise (microphone)"
+286,/m/0jb2l,"Thunderstorm"
+287,/m/0ngt1,"Thunder"
+288,/m/0838f,"Water"
+289,/m/06mb1,"Rain"
+290,/m/07r10fb,"Raindrop"
+291,/t/dd00038,"Rain on surface"
+292,/m/0j6m2,"Stream"
+293,/m/0j2kx,"Waterfall"
+294,/m/05kq4,"Ocean"
+295,/m/034srq,"Waves, surf"
+296,/m/06wzb,"Steam"
+297,/m/07swgks,"Gurgling"
+298,/m/02_41,"Fire"
+299,/m/07pzfmf,"Crackle"
+300,/m/07yv9,"Vehicle"
+301,/m/019jd,"Boat, Water vehicle"
+302,/m/0hsrw,"Sailboat, sailing ship"
+303,/m/056ks2,"Rowboat, canoe, kayak"
+304,/m/02rlv9,"Motorboat, speedboat"
+305,/m/06q74,"Ship"
+306,/m/012f08,"Motor vehicle (road)"
+307,/m/0k4j,"Car"
+308,/m/0912c9,"Vehicle horn, car horn, honking"
+309,/m/07qv_d5,"Toot"
+310,/m/02mfyn,"Car alarm"
+311,/m/04gxbd,"Power windows, electric windows"
+312,/m/07rknqz,"Skidding"
+313,/m/0h9mv,"Tire squeal"
+314,/t/dd00134,"Car passing by"
+315,/m/0ltv,"Race car, auto racing"
+316,/m/07r04,"Truck"
+317,/m/0gvgw0,"Air brake"
+318,/m/05x_td,"Air horn, truck horn"
+319,/m/02rhddq,"Reversing beeps"
+320,/m/03cl9h,"Ice cream truck, ice cream van"
+321,/m/01bjv,"Bus"
+322,/m/03j1ly,"Emergency vehicle"
+323,/m/04qvtq,"Police car (siren)"
+324,/m/012n7d,"Ambulance (siren)"
+325,/m/012ndj,"Fire engine, fire truck (siren)"
+326,/m/04_sv,"Motorcycle"
+327,/m/0btp2,"Traffic noise, roadway noise"
+328,/m/06d_3,"Rail transport"
+329,/m/07jdr,"Train"
+330,/m/04zmvq,"Train whistle"
+331,/m/0284vy3,"Train horn"
+332,/m/01g50p,"Railroad car, train wagon"
+333,/t/dd00048,"Train wheels squealing"
+334,/m/0195fx,"Subway, metro, underground"
+335,/m/0k5j,"Aircraft"
+336,/m/014yck,"Aircraft engine"
+337,/m/04229,"Jet engine"
+338,/m/02l6bg,"Propeller, airscrew"
+339,/m/09ct_,"Helicopter"
+340,/m/0cmf2,"Fixed-wing aircraft, airplane"
+341,/m/0199g,"Bicycle"
+342,/m/06_fw,"Skateboard"
+343,/m/02mk9,"Engine"
+344,/t/dd00065,"Light engine (high frequency)"
+345,/m/08j51y,"Dental drill, dentist's drill"
+346,/m/01yg9g,"Lawn mower"
+347,/m/01j4z9,"Chainsaw"
+348,/t/dd00066,"Medium engine (mid frequency)"
+349,/t/dd00067,"Heavy engine (low frequency)"
+350,/m/01h82_,"Engine knocking"
+351,/t/dd00130,"Engine starting"
+352,/m/07pb8fc,"Idling"
+353,/m/07q2z82,"Accelerating, revving, vroom"
+354,/m/02dgv,"Door"
+355,/m/03wwcy,"Doorbell"
+356,/m/07r67yg,"Ding-dong"
+357,/m/02y_763,"Sliding door"
+358,/m/07rjzl8,"Slam"
+359,/m/07r4wb8,"Knock"
+360,/m/07qcpgn,"Tap"
+361,/m/07q6cd_,"Squeak"
+362,/m/0642b4,"Cupboard open or close"
+363,/m/0fqfqc,"Drawer open or close"
+364,/m/04brg2,"Dishes, pots, and pans"
+365,/m/023pjk,"Cutlery, silverware"
+366,/m/07pn_8q,"Chopping (food)"
+367,/m/0dxrf,"Frying (food)"
+368,/m/0fx9l,"Microwave oven"
+369,/m/02pjr4,"Blender"
+370,/m/02jz0l,"Water tap, faucet"
+371,/m/0130jx,"Sink (filling or washing)"
+372,/m/03dnzn,"Bathtub (filling or washing)"
+373,/m/03wvsk,"Hair dryer"
+374,/m/01jt3m,"Toilet flush"
+375,/m/012xff,"Toothbrush"
+376,/m/04fgwm,"Electric toothbrush"
+377,/m/0d31p,"Vacuum cleaner"
+378,/m/01s0vc,"Zipper (clothing)"
+379,/m/03v3yw,"Keys jangling"
+380,/m/0242l,"Coin (dropping)"
+381,/m/01lsmm,"Scissors"
+382,/m/02g901,"Electric shaver, electric razor"
+383,/m/05rj2,"Shuffling cards"
+384,/m/0316dw,"Typing"
+385,/m/0c2wf,"Typewriter"
+386,/m/01m2v,"Computer keyboard"
+387,/m/081rb,"Writing"
+388,/m/07pp_mv,"Alarm"
+389,/m/07cx4,"Telephone"
+390,/m/07pp8cl,"Telephone bell ringing"
+391,/m/01hnzm,"Ringtone"
+392,/m/02c8p,"Telephone dialing, DTMF"
+393,/m/015jpf,"Dial tone"
+394,/m/01z47d,"Busy signal"
+395,/m/046dlr,"Alarm clock"
+396,/m/03kmc9,"Siren"
+397,/m/0dgbq,"Civil defense siren"
+398,/m/030rvx,"Buzzer"
+399,/m/01y3hg,"Smoke detector, smoke alarm"
+400,/m/0c3f7m,"Fire alarm"
+401,/m/04fq5q,"Foghorn"
+402,/m/0l156k,"Whistle"
+403,/m/06hck5,"Steam whistle"
+404,/t/dd00077,"Mechanisms"
+405,/m/02bm9n,"Ratchet, pawl"
+406,/m/01x3z,"Clock"
+407,/m/07qjznt,"Tick"
+408,/m/07qjznl,"Tick-tock"
+409,/m/0l7xg,"Gears"
+410,/m/05zc1,"Pulleys"
+411,/m/0llzx,"Sewing machine"
+412,/m/02x984l,"Mechanical fan"
+413,/m/025wky1,"Air conditioning"
+414,/m/024dl,"Cash register"
+415,/m/01m4t,"Printer"
+416,/m/0dv5r,"Camera"
+417,/m/07bjf,"Single-lens reflex camera"
+418,/m/07k1x,"Tools"
+419,/m/03l9g,"Hammer"
+420,/m/03p19w,"Jackhammer"
+421,/m/01b82r,"Sawing"
+422,/m/02p01q,"Filing (rasp)"
+423,/m/023vsd,"Sanding"
+424,/m/0_ksk,"Power tool"
+425,/m/01d380,"Drill"
+426,/m/014zdl,"Explosion"
+427,/m/032s66,"Gunshot, gunfire"
+428,/m/04zjc,"Machine gun"
+429,/m/02z32qm,"Fusillade"
+430,/m/0_1c,"Artillery fire"
+431,/m/073cg4,"Cap gun"
+432,/m/0g6b5,"Fireworks"
+433,/g/122z_qxw,"Firecracker"
+434,/m/07qsvvw,"Burst, pop"
+435,/m/07pxg6y,"Eruption"
+436,/m/07qqyl4,"Boom"
+437,/m/083vt,"Wood"
+438,/m/07pczhz,"Chop"
+439,/m/07pl1bw,"Splinter"
+440,/m/07qs1cx,"Crack"
+441,/m/039jq,"Glass"
+442,/m/07q7njn,"Chink, clink"
+443,/m/07rn7sz,"Shatter"
+444,/m/04k94,"Liquid"
+445,/m/07rrlb6,"Splash, splatter"
+446,/m/07p6mqd,"Slosh"
+447,/m/07qlwh6,"Squish"
+448,/m/07r5v4s,"Drip"
+449,/m/07prgkl,"Pour"
+450,/m/07pqc89,"Trickle, dribble"
+451,/t/dd00088,"Gush"
+452,/m/07p7b8y,"Fill (with liquid)"
+453,/m/07qlf79,"Spray"
+454,/m/07ptzwd,"Pump (liquid)"
+455,/m/07ptfmf,"Stir"
+456,/m/0dv3j,"Boiling"
+457,/m/0790c,"Sonar"
+458,/m/0dl83,"Arrow"
+459,/m/07rqsjt,"Whoosh, swoosh, swish"
+460,/m/07qnq_y,"Thump, thud"
+461,/m/07rrh0c,"Thunk"
+462,/m/0b_fwt,"Electronic tuner"
+463,/m/02rr_,"Effects unit"
+464,/m/07m2kt,"Chorus effect"
+465,/m/018w8,"Basketball bounce"
+466,/m/07pws3f,"Bang"
+467,/m/07ryjzk,"Slap, smack"
+468,/m/07rdhzs,"Whack, thwack"
+469,/m/07pjjrj,"Smash, crash"
+470,/m/07pc8lb,"Breaking"
+471,/m/07pqn27,"Bouncing"
+472,/m/07rbp7_,"Whip"
+473,/m/07pyf11,"Flap"
+474,/m/07qb_dv,"Scratch"
+475,/m/07qv4k0,"Scrape"
+476,/m/07pdjhy,"Rub"
+477,/m/07s8j8t,"Roll"
+478,/m/07plct2,"Crushing"
+479,/t/dd00112,"Crumpling, crinkling"
+480,/m/07qcx4z,"Tearing"
+481,/m/02fs_r,"Beep, bleep"
+482,/m/07qwdck,"Ping"
+483,/m/07phxs1,"Ding"
+484,/m/07rv4dm,"Clang"
+485,/m/07s02z0,"Squeal"
+486,/m/07qh7jl,"Creak"
+487,/m/07qwyj0,"Rustle"
+488,/m/07s34ls,"Whir"
+489,/m/07qmpdm,"Clatter"
+490,/m/07p9k1k,"Sizzle"
+491,/m/07qc9xj,"Clicking"
+492,/m/07rwm0c,"Clickety-clack"
+493,/m/07phhsh,"Rumble"
+494,/m/07qyrcz,"Plop"
+495,/m/07qfgpx,"Jingle, tinkle"
+496,/m/07rcgpl,"Hum"
+497,/m/07p78v5,"Zing"
+498,/t/dd00121,"Boing"
+499,/m/07s12q4,"Crunch"
+500,/m/028v0c,"Silence"
+501,/m/01v_m0,"Sine wave"
+502,/m/0b9m1,"Harmonic"
+503,/m/0hdsk,"Chirp tone"
+504,/m/0c1dj,"Sound effect"
+505,/m/07pt_g0,"Pulse"
+506,/t/dd00125,"Inside, small room"
+507,/t/dd00126,"Inside, large room or hall"
+508,/t/dd00127,"Inside, public space"
+509,/t/dd00128,"Outside, urban or manmade"
+510,/t/dd00129,"Outside, rural or natural"
+511,/m/01b9nn,"Reverberation"
+512,/m/01jnbd,"Echo"
+513,/m/096m7z,"Noise"
+514,/m/06_y0by,"Environmental noise"
+515,/m/07rgkc5,"Static"
+516,/m/06xkwv,"Mains hum"
+517,/m/0g12c5,"Distortion"
+518,/m/08p9q4,"Sidetone"
+519,/m/07szfh9,"Cacophony"
+520,/m/0chx_,"White noise"
+521,/m/0cj0r,"Pink noise"
+522,/m/07p_0gm,"Throbbing"
+523,/m/01jwx6,"Vibration"
+524,/m/07c52,"Television"
+525,/m/06bz3,"Radio"
+526,/m/07hvw1,"Field recording"

RAG/Knowledge_Database/languagebind_main/a_cls/dataloader.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+# @Time    : 6/19/21 12:23 AM
+# @Author  : Yuan Gong
+# @Affiliation  : Massachusetts Institute of Technology
+# @Email   : yuangong@mit.edu
+# @File    : dataloader.py
+
+# modified from:
+# Author: David Harwath
+# with some functions borrowed from https://github.com/SeanNaren/deepspeech.pytorch
+
+import csv
+import json
+import logging
+
+import torchaudio
+import numpy as np
+import torch
+import torch.nn.functional
+from torch.utils.data import Dataset
+import random
+
+def make_midname_dict(label_csv):
+    index_lookup = {}
+    with open(label_csv, 'r') as f:
+        csv_reader = csv.DictReader(f)
+        line_count = 0
+        for row in csv_reader:
+            index_lookup[row['mid']] = row['display_name']
+            line_count += 1
+    return index_lookup
+
+def make_index_dict(label_csv):
+    index_lookup = {}
+    with open(label_csv, 'r') as f:
+        csv_reader = csv.DictReader(f)
+        line_count = 0
+        for row in csv_reader:
+            index_lookup[row['mid']] = row['index']
+            line_count += 1
+    return index_lookup
+
+def make_name_dict(label_csv):
+    name_lookup = {}
+    with open(label_csv, 'r') as f:
+        csv_reader = csv.DictReader(f)
+        line_count = 0
+        for row in csv_reader:
+            name_lookup[row['index']] = row['display_name']
+            line_count += 1
+    return name_lookup
+
+def lookup_list(index_list, label_csv):
+    label_list = []
+    table = make_name_dict(label_csv)
+    for item in index_list:
+        label_list.append(table[item])
+    return label_list
+
+def preemphasis(signal,coeff=0.97):
+    """perform preemphasis on the input signal.
+
+    :param signal: The signal to filter.
+    :param coeff: The preemphasis coefficient. 0 is none, default 0.97.
+    :returns: the filtered signal.
+    """
+    return np.append(signal[0],signal[1:]-coeff*signal[:-1])
+
+class AudiosetDataset(Dataset):
+    def __init__(self, dataset_json_file, audio_conf, label_csv=None):
+        """
+        Dataset that manages audio recordings
+        :param audio_conf: Dictionary containing the audio loading and preprocessing settings
+        :param dataset_json_file
+        """
+        self.datapath = dataset_json_file
+        with open(dataset_json_file, 'r') as fp:
+            data_json = json.load(fp)
+        self.data = data_json['data']
+        self.index_dict = make_index_dict(label_csv)
+        self.label_num = len(self.index_dict)
+
+    def __getitem__(self, index):
+        datum = self.data[index]
+        label_indices = np.zeros(self.label_num)
+        try:
+            fbank, mix_lambda = self._wav2fbank(datum['wav'])
+        except Exception as e:
+            logging.warning(f"Error at {datum['wav']} with \"{e}\"")
+            return self.__getitem__(random.randint(0, self.__len__()-1))
+        for label_str in datum['labels'].split(','):
+            label_indices[int(self.index_dict[label_str])] = 1.0
+
+        label_indices = torch.FloatTensor(label_indices)
+
+
+        return fbank, label_indices
+
+    def __len__(self):
+        return len(self.data)

RAG/Knowledge_Database/languagebind_main/a_cls/datasets.py

@@ -0,0 +1,93 @@
+import os.path
+
+import torch
+
+from data.build_datasets import DataInfo
+from data.process_audio import get_audio_transform, torchaudio_loader
+from torchvision import datasets
+
+# -*- coding: utf-8 -*-
+# @Time    : 6/19/21 12:23 AM
+# @Author  : Yuan Gong
+# @Affiliation  : Massachusetts Institute of Technology
+# @Email   : yuangong@mit.edu
+# @File    : dataloader.py
+
+# modified from:
+# Author: David Harwath
+# with some functions borrowed from https://github.com/SeanNaren/deepspeech.pytorch
+
+import csv
+import json
+import logging
+
+import torchaudio
+import numpy as np
+import torch
+import torch.nn.functional
+from torch.utils.data import Dataset
+import random
+
+
+def make_index_dict(label_csv):
+    index_lookup = {}
+    with open(label_csv, 'r') as f:
+        csv_reader = csv.DictReader(f)
+        line_count = 0
+        for row in csv_reader:
+            index_lookup[row['mid']] = row['index']
+            line_count += 1
+    return index_lookup
+
+
+class AudiosetDataset(Dataset):
+    def __init__(self, args, transform, loader):
+        self.audio_root = '/apdcephfs_cq3/share_1311970/downstream_datasets/Audio/audioset/eval_segments'
+        dataset_json_file = '/apdcephfs_cq3/share_1311970/downstream_datasets/Audio/audioset/filter_eval.json'
+        label_csv = '/apdcephfs_cq3/share_1311970/downstream_datasets/Audio/audioset/class_labels_indices.csv'
+        with open(dataset_json_file, 'r') as fp:
+            data_json = json.load(fp)
+        self.data = data_json['data']
+        self.index_dict = make_index_dict(label_csv)
+        self.label_num = len(self.index_dict)
+
+        self.args = args
+        self.transform = transform
+        self.loader = loader
+
+    def __getitem__(self, index):
+        datum = self.data[index]
+        label_indices = np.zeros(self.label_num)
+        for label_str in datum['labels'].split(','):
+            label_indices[int(self.index_dict[label_str])] = 1.0
+        label_indices = torch.FloatTensor(label_indices)
+
+        audio = self.loader(os.path.join(self.audio_root, datum['wav']))
+        audio_data = self.transform(audio)
+        return audio_data, label_indices
+
+    def __len__(self):
+        return len(self.data)
+
+
+
+def is_valid_file(path):
+    return True
+
+def get_audio_dataset(args):
+    data_path = args.audio_data_path
+    transform = get_audio_transform(args)
+
+    if args.val_a_cls_data.lower() == 'audioset':
+        dataset = AudiosetDataset(args, transform=transform, loader=torchaudio_loader)
+    else:
+        dataset = datasets.ImageFolder(data_path, transform=transform, loader=torchaudio_loader, is_valid_file=is_valid_file)
+
+    dataloader = torch.utils.data.DataLoader(
+        dataset,
+        batch_size=args.batch_size,
+        num_workers=args.workers,
+        sampler=None,
+    )
+
+    return DataInfo(dataloader=dataloader, sampler=None)

RAG/Knowledge_Database/languagebind_main/a_cls/filter_eval_audio.py

@@ -0,0 +1,21 @@
+import json
+import os.path
+from tqdm import tqdm
+
+with open(r"G:\audioset\audioset\zip_audios\16k\eval.json", 'r') as f:
+    data = json.load(f)['data']
+
+new_data = []
+total = 0
+success = 0
+for i in tqdm(data):
+    total += 1
+    video_id = os.path.basename(i['wav'])
+    new_video_id = 'Y' + video_id
+    i['wav'] = new_video_id
+    if os.path.exists(f"G:/audioset/audioset/zip_audios/eval_segments/{i['wav']}") and not video_id.startswith('mW3S0u8bj58'):
+        new_data.append(i)
+        success += 1
+print(total, success, total-success)
+with open(r"G:\audioset\audioset\zip_audios\16k\filter_eval.json", 'w') as f:
+    data = json.dump({'data': new_data}, f, indent=2)

RAG/Knowledge_Database/languagebind_main/a_cls/precision.py

@@ -0,0 +1,12 @@
+import torch
+from contextlib import suppress
+
+
+def get_autocast(precision):
+    if precision == 'amp':
+        return torch.cuda.amp.autocast
+    elif precision == 'amp_bfloat16' or precision == 'amp_bf16':
+        # amp_bfloat16 is more stable than amp float16 for clip training
+        return lambda: torch.cuda.amp.autocast(dtype=torch.bfloat16)
+    else:
+        return suppress

RAG/Knowledge_Database/languagebind_main/a_cls/stats.py

@@ -0,0 +1,57 @@
+import numpy as np
+from scipy import stats
+from sklearn import metrics
+import torch
+
+def d_prime(auc):
+    standard_normal = stats.norm()
+    d_prime = standard_normal.ppf(auc) * np.sqrt(2.0)
+    return d_prime
+
+def calculate_stats(output, target):
+    """Calculate statistics including mAP, AUC, etc.
+
+    Args:
+      output: 2d array, (samples_num, classes_num)
+      target: 2d array, (samples_num, classes_num)
+
+    Returns:
+      stats: list of statistic of each class.
+    """
+
+    classes_num = target.shape[-1]
+    stats = []
+
+    # Accuracy, only used for single-label classification such as esc-50, not for multiple label one such as AudioSet
+    acc = metrics.accuracy_score(np.argmax(target, 1), np.argmax(output, 1))
+
+    # Class-wise statistics
+    for k in range(classes_num):
+
+        # Average precision
+        avg_precision = metrics.average_precision_score(
+            target[:, k], output[:, k], average=None)
+
+        # AUC
+        auc = metrics.roc_auc_score(target[:, k], output[:, k], average=None)
+
+        # Precisions, recalls
+        (precisions, recalls, thresholds) = metrics.precision_recall_curve(
+            target[:, k], output[:, k])
+
+        # FPR, TPR
+        (fpr, tpr, thresholds) = metrics.roc_curve(target[:, k], output[:, k])
+
+        save_every_steps = 1000     # Sample statistics to reduce size
+        dict = {'precisions': precisions[0::save_every_steps],
+                'recalls': recalls[0::save_every_steps],
+                'AP': avg_precision,
+                'fpr': fpr[0::save_every_steps],
+                'fnr': 1. - tpr[0::save_every_steps],
+                'auc': auc,
+                # note acc is not class-wise, this is just to keep consistent with other metrics
+                'acc': acc
+                }
+        stats.append(dict)
+
+    return stats

RAG/Knowledge_Database/languagebind_main/a_cls/util.py

@@ -0,0 +1,306 @@
+import math
+import pickle
+import numpy as np
+import torch
+import torch.nn as nn
+import random
+from collections import namedtuple
+
+def calc_recalls(S):
+    """
+    Computes recall at 1, 5, and 10 given a similarity matrix S.
+    By convention, rows of S are assumed to correspond to images and columns are captions.
+    """
+    assert(S.dim() == 2)
+    assert(S.size(0) == S.size(1))
+    if isinstance(S, torch.autograd.Variable):
+        S = S.data
+    n = S.size(0)
+    A2I_scores, A2I_ind = S.topk(10, 0)
+    I2A_scores, I2A_ind = S.topk(10, 1)
+    A_r1 = AverageMeter()
+    A_r5 = AverageMeter()
+    A_r10 = AverageMeter()
+    I_r1 = AverageMeter()
+    I_r5 = AverageMeter()
+    I_r10 = AverageMeter()
+    for i in range(n):
+        A_foundind = -1
+        I_foundind = -1
+        for ind in range(10):
+            if A2I_ind[ind, i] == i:
+                I_foundind = ind
+            if I2A_ind[i, ind] == i:
+                A_foundind = ind
+        # do r1s
+        if A_foundind == 0:
+            A_r1.update(1)
+        else:
+            A_r1.update(0)
+        if I_foundind == 0:
+            I_r1.update(1)
+        else:
+            I_r1.update(0)
+        # do r5s
+        if A_foundind >= 0 and A_foundind < 5:
+            A_r5.update(1)
+        else:
+            A_r5.update(0)
+        if I_foundind >= 0 and I_foundind < 5:
+            I_r5.update(1)
+        else:
+            I_r5.update(0)
+        # do r10s
+        if A_foundind >= 0 and A_foundind < 10:
+            A_r10.update(1)
+        else:
+            A_r10.update(0)
+        if I_foundind >= 0 and I_foundind < 10:
+            I_r10.update(1)
+        else:
+            I_r10.update(0)
+
+    recalls = {'A_r1':A_r1.avg, 'A_r5':A_r5.avg, 'A_r10':A_r10.avg,
+                'I_r1':I_r1.avg, 'I_r5':I_r5.avg, 'I_r10':I_r10.avg}
+                #'A_meanR':A_meanR.avg, 'I_meanR':I_meanR.avg}
+
+    return recalls
+
+def computeMatchmap(I, A):
+    assert(I.dim() == 3)
+    assert(A.dim() == 2)
+    D = I.size(0)
+    H = I.size(1)
+    W = I.size(2)
+    T = A.size(1)
+    Ir = I.view(D, -1).t()
+    matchmap = torch.mm(Ir, A)
+    matchmap = matchmap.view(H, W, T)
+    return matchmap
+
+def matchmapSim(M, simtype):
+    assert(M.dim() == 3)
+    if simtype == 'SISA':
+        return M.mean()
+    elif simtype == 'MISA':
+        M_maxH, _ = M.max(0)
+        M_maxHW, _ = M_maxH.max(0)
+        return M_maxHW.mean()
+    elif simtype == 'SIMA':
+        M_maxT, _ = M.max(2)
+        return M_maxT.mean()
+    else:
+        raise ValueError
+
+def sampled_margin_rank_loss(image_outputs, audio_outputs, nframes, margin=1., simtype='MISA'):
+    """
+    Computes the triplet margin ranking loss for each anchor image/caption pair
+    The impostor image/caption is randomly sampled from the minibatch
+    """
+    assert(image_outputs.dim() == 4)
+    assert(audio_outputs.dim() == 3)
+    n = image_outputs.size(0)
+    loss = torch.zeros(1, device=image_outputs.device, requires_grad=True)
+    for i in range(n):
+        I_imp_ind = i
+        A_imp_ind = i
+        while I_imp_ind == i:
+            I_imp_ind = np.random.randint(0, n)
+        while A_imp_ind == i:
+            A_imp_ind = np.random.randint(0, n)
+        nF = nframes[i]
+        nFimp = nframes[A_imp_ind]
+        anchorsim = matchmapSim(computeMatchmap(image_outputs[i], audio_outputs[i][:, 0:nF]), simtype)
+        Iimpsim = matchmapSim(computeMatchmap(image_outputs[I_imp_ind], audio_outputs[i][:, 0:nF]), simtype)
+        Aimpsim = matchmapSim(computeMatchmap(image_outputs[i], audio_outputs[A_imp_ind][:, 0:nFimp]), simtype)
+        A2I_simdif = margin + Iimpsim - anchorsim
+        if (A2I_simdif.data > 0).all():
+            loss = loss + A2I_simdif
+        I2A_simdif = margin + Aimpsim - anchorsim
+        if (I2A_simdif.data > 0).all():
+            loss = loss + I2A_simdif
+    loss = loss / n
+    return loss
+
+def compute_matchmap_similarity_matrix(image_outputs, audio_outputs, nframes, simtype='MISA'):
+    """
+    Assumes image_outputs is a (batchsize, embedding_dim, rows, height) tensor
+    Assumes audio_outputs is a (batchsize, embedding_dim, 1, time) tensor
+    Returns similarity matrix S where images are rows and audios are along the columns
+    """
+    assert(image_outputs.dim() == 4)
+    assert(audio_outputs.dim() == 3)
+    n = image_outputs.size(0)
+    S = torch.zeros(n, n, device=image_outputs.device)
+    for image_idx in range(n):
+            for audio_idx in range(n):
+                nF = max(1, nframes[audio_idx])
+                S[image_idx, audio_idx] = matchmapSim(computeMatchmap(image_outputs[image_idx], audio_outputs[audio_idx][:, 0:nF]), simtype)
+    return S
+
+def compute_pooldot_similarity_matrix(image_outputs, audio_outputs, nframes):
+    """
+    Assumes image_outputs is a (batchsize, embedding_dim, rows, height) tensor
+    Assumes audio_outputs is a (batchsize, embedding_dim, 1, time) tensor
+    Returns similarity matrix S where images are rows and audios are along the columns
+    S[i][j] is computed as the dot product between the meanpooled embeddings of
+    the ith image output and jth audio output
+    """
+    assert(image_outputs.dim() == 4)
+    assert(audio_outputs.dim() == 4)
+    n = image_outputs.size(0)
+    imagePoolfunc = nn.AdaptiveAvgPool2d((1, 1))
+    pooled_image_outputs = imagePoolfunc(image_outputs).squeeze(3).squeeze(2)
+    audioPoolfunc = nn.AdaptiveAvgPool2d((1, 1))
+    pooled_audio_outputs_list = []
+    for idx in range(n):
+        nF = max(1, nframes[idx])
+        pooled_audio_outputs_list.append(audioPoolfunc(audio_outputs[idx][:, :, 0:nF]).unsqueeze(0))
+    pooled_audio_outputs = torch.cat(pooled_audio_outputs_list).squeeze(3).squeeze(2)
+    S = torch.mm(pooled_image_outputs, pooled_audio_outputs.t())
+    return S
+
+def one_imposter_index(i, N):
+    imp_ind = random.randint(0, N - 2)
+    if imp_ind == i:
+        imp_ind = N - 1
+    return imp_ind
+
+def basic_get_imposter_indices(N):
+    imposter_idc = []
+    for i in range(N):
+        # Select an imposter index for example i:
+        imp_ind = one_imposter_index(i, N)
+        imposter_idc.append(imp_ind)
+    return imposter_idc
+
+def semihardneg_triplet_loss_from_S(S, margin):
+    """
+    Input: Similarity matrix S as an autograd.Variable
+    Output: The one-way triplet loss from rows of S to columns of S. Impostors are taken
+    to be the most similar point to the anchor that is still less similar to the anchor
+    than the positive example.
+    You would need to run this function twice, once with S and once with S.t(),
+    in order to compute the triplet loss in both directions.
+    """
+    assert(S.dim() == 2)
+    assert(S.size(0) == S.size(1))
+    N = S.size(0)
+    loss = torch.autograd.Variable(torch.zeros(1).type(S.data.type()), requires_grad=True)
+    # Imposter - ground truth
+    Sdiff = S - torch.diag(S).view(-1, 1)
+    eps = 1e-12
+    # All examples less similar than ground truth
+    mask = (Sdiff < -eps).type(torch.LongTensor)
+    maskf = mask.type_as(S)
+    # Mask out all examples >= gt with minimum similarity
+    Sp = maskf * Sdiff + (1 - maskf) * torch.min(Sdiff).detach()
+    # Find the index maximum similar of the remaining
+    _, idc = Sp.max(dim=1)
+    idc = idc.data.cpu()
+    # Vector mask: 1 iff there exists an example < gt
+    has_neg = (mask.sum(dim=1) > 0).data.type(torch.LongTensor)
+    # Random imposter indices
+    random_imp_ind = torch.LongTensor(basic_get_imposter_indices(N))
+    # Use hardneg if there exists an example < gt, otherwise use random imposter
+    imp_idc = has_neg * idc + (1 - has_neg) * random_imp_ind
+    # This could probably be vectorized too, but I haven't.
+    for i, imp in enumerate(imp_idc):
+        local_loss = Sdiff[i, imp] + margin
+        if (local_loss.data > 0).all():
+            loss = loss + local_loss
+    loss = loss / N
+    return loss
+
+def sampled_triplet_loss_from_S(S, margin):
+    """
+    Input: Similarity matrix S as an autograd.Variable
+    Output: The one-way triplet loss from rows of S to columns of S. Imposters are
+    randomly sampled from the columns of S.
+    You would need to run this function twice, once with S and once with S.t(),
+    in order to compute the triplet loss in both directions.
+    """
+    assert(S.dim() == 2)
+    assert(S.size(0) == S.size(1))
+    N = S.size(0)
+    loss = torch.autograd.Variable(torch.zeros(1).type(S.data.type()), requires_grad=True)
+    # Imposter - ground truth
+    Sdiff = S - torch.diag(S).view(-1, 1)
+    imp_ind = torch.LongTensor(basic_get_imposter_indices(N))
+    # This could probably be vectorized too, but I haven't.
+    for i, imp in enumerate(imp_ind):
+        local_loss = Sdiff[i, imp] + margin
+        if (local_loss.data > 0).all():
+            loss = loss + local_loss
+    loss = loss / N
+    return loss
+
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+def adjust_learning_rate(base_lr, lr_decay, optimizer, epoch):
+    """Sets the learning rate to the initial LR decayed by 10 every lr_decay epochs"""
+    lr = base_lr * (0.1 ** (epoch // lr_decay))
+    print('now learning rate changed to {:f}'.format(lr))
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = lr
+
+def adjust_learning_rate2(base_lr, lr_decay, optimizer, epoch):
+    """Sets the learning rate to the initial LR decayed by 10 every lr_decay epochs"""
+    for param_group in optimizer.param_groups:
+        cur_lr = param_group['lr']
+        print('current learing rate is {:f}'.format(lr))
+    lr = cur_lr  * 0.1
+    print('now learning rate changed to {:f}'.format(lr))
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = lr
+
+
+def load_progress(prog_pkl, quiet=False):
+    """
+    load progress pkl file
+    Args:
+        prog_pkl(str): path to progress pkl file
+    Return:
+        progress(list):
+        epoch(int):
+        global_step(int):
+        best_epoch(int):
+        best_avg_r10(float):
+    """
+    def _print(msg):
+        if not quiet:
+            print(msg)
+
+    with open(prog_pkl, "rb") as f:
+        prog = pickle.load(f)
+        epoch, global_step, best_epoch, best_avg_r10, _ = prog[-1]
+
+    _print("\nPrevious Progress:")
+    msg =  "[%5s %7s %5s %7s %6s]" % ("epoch", "step", "best_epoch", "best_avg_r10", "time")
+    _print(msg)
+    return prog, epoch, global_step, best_epoch, best_avg_r10
+
+def count_parameters(model):
+    return sum([p.numel() for p in model.parameters() if p.requires_grad])
+
+PrenetConfig = namedtuple(
+  'PrenetConfig', ['input_size', 'hidden_size', 'num_layers', 'dropout'])
+
+RNNConfig = namedtuple(
+  'RNNConfig',
+  ['input_size', 'hidden_size', 'num_layers', 'dropout', 'residual'])

RAG/Knowledge_Database/languagebind_main/a_cls/zero_shot.py

@@ -0,0 +1,234 @@
+import logging
+import os
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+from tqdm import tqdm
+
+from open_clip import get_input_dtype, get_tokenizer
+from open_clip.factory import HF_HUB_PREFIX
+from .precision import get_autocast
+from .stats import calculate_stats, d_prime
+from .zero_shot_classifier import build_zero_shot_classifier
+from .zero_shot_metadata import CLASSNAMES, OPENAI_IMAGENET_TEMPLATES
+
+
+def accuracy(output, target, topk=(1,)):
+    pred = output.topk(max(topk), 1, True, True)[1].t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+    return [float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) for k in topk]
+
+
+def run(model, classifier, dataloader, args):
+    autocast = get_autocast(args.precision)
+    input_dtype = get_input_dtype(args.precision)
+
+    with torch.no_grad():
+        top1, top5, n = 0., 0., 0.
+        for images, target in tqdm(dataloader, unit_scale=args.batch_size):
+            images = images.to(device=args.device, dtype=input_dtype)
+            images = images.unsqueeze(2)
+            target = target.to(args.device)
+
+            with autocast():
+                # predict
+                output = model(image=images)
+                image_features = output['image_features'] if isinstance(output, dict) else output[0]
+                logits = 100. * image_features @ classifier
+
+            # measure accuracy
+            acc1, acc5 = accuracy(logits, target, topk=(1, 5))
+            top1 += acc1
+            top5 += acc5
+            n += images.size(0)
+
+    top1 = (top1 / n)
+    top5 = (top5 / n)
+    return top1, top5
+
+
+def validate(audio_model, classifier, val_loader, args, epoch):
+    epoch = epoch - 1 ########################
+    # switch to evaluate mode
+    audio_model.eval()
+    autocast = get_autocast(args.precision)
+    input_dtype = get_input_dtype(args.precision)
+    A_predictions = []
+    A_targets = []
+    A_loss = []
+    with torch.no_grad():
+        for i, (audio_input, labels) in enumerate(tqdm(val_loader)):
+            audio_input = audio_input.to(device=args.device, dtype=input_dtype)
+
+            # compute output
+            with autocast():
+                # predict
+                output = audio_model(image=audio_input)
+                image_features = output['image_features'] if isinstance(output, dict) else output[0]
+                logits = 100. * image_features @ classifier
+            audio_output = logits
+
+            # audio_output = torch.sigmoid(audio_output)
+            predictions = audio_output.to('cpu').detach()
+
+            A_predictions.append(predictions)
+            A_targets.append(labels)
+
+            # compute the loss
+            labels = labels.to(args.device)
+            loss = nn.CrossEntropyLoss()(audio_output, torch.argmax(labels.long(), dim=1))
+            A_loss.append(loss.to('cpu').detach())
+
+        audio_output = torch.cat(A_predictions)
+        target = torch.cat(A_targets)
+        loss = np.mean(A_loss)
+        stats = calculate_stats(audio_output, target)
+
+        # save the prediction here
+        args.a_cls_output_dir = os.path.join(args.log_base_path, f'a_cls/{args.val_a_cls_data.lower()}')
+        os.makedirs(args.a_cls_output_dir, exist_ok=True)
+        if os.path.exists(args.a_cls_output_dir + '/predictions') == False:
+            os.mkdir(args.a_cls_output_dir + '/predictions')
+            np.savetxt(args.a_cls_output_dir + '/predictions/target.csv', target, delimiter=',')
+        np.savetxt(args.a_cls_output_dir + '/predictions/predictions_' + str(epoch) + '.csv', audio_output,
+                   delimiter=',')
+
+    valid_loss = loss
+    main_metrics = 'mAP'
+    metrics = {}
+
+    if args.do_train:
+        # ensemble results
+        cum_stats = validate_ensemble(args, epoch)
+        cum_mAP = np.mean([stat['AP'] for stat in cum_stats])
+        cum_mAUC = np.mean([stat['auc'] for stat in cum_stats])
+        cum_acc = cum_stats[0]['acc']
+
+    mAP = np.mean([stat['AP'] for stat in stats])
+    mAUC = np.mean([stat['auc'] for stat in stats])
+    acc = stats[0]['acc']
+
+    middle_ps = [stat['precisions'][int(len(stat['precisions']) / 2)] for stat in stats]
+    middle_rs = [stat['recalls'][int(len(stat['recalls']) / 2)] for stat in stats]
+    average_precision = np.mean(middle_ps)
+    average_recall = np.mean(middle_rs)
+
+    if main_metrics == 'mAP':
+        logging.info("mAP: {:.6f}".format(mAP))
+    else:
+        logging.info("acc: {:.6f}".format(acc))
+    logging.info("AUC: {:.6f}".format(mAUC))
+    logging.info("Avg Precision: {:.6f}".format(average_precision))
+    logging.info("Avg Recall: {:.6f}".format(average_recall))
+    logging.info("d_prime: {:.6f}".format(d_prime(mAUC)))
+    logging.info("valid_loss: {:.6f}".format(valid_loss))
+
+    if args.do_train:
+        logging.info("cum_mAP: {:.6f}".format(cum_mAP))
+        logging.info("cum_mAUC: {:.6f}".format(cum_mAUC))
+
+    if main_metrics == 'mAP':
+        metrics['mAP'] = float(mAP)
+    else:
+        metrics['acc'] = float(acc)
+
+    metrics['mAUC'] = float(mAUC)
+    metrics['average_precision'] = float(average_precision)
+    metrics['average_recall'] = float(average_recall)
+    metrics['d_prime_mAUC'] = float(d_prime(mAUC))
+    metrics['valid_loss'] = float(valid_loss)
+
+    if args.do_train:
+        metrics['cum_mAP'] = float(cum_mAP)
+        metrics['cum_mAUC'] = float(cum_mAUC)
+
+    return metrics
+
+
+def validate_ensemble(args, epoch):
+    exp_dir = args.a_cls_output_dir
+    target = np.loadtxt(exp_dir + '/predictions/target.csv', delimiter=',')
+    if epoch == 0:
+        cum_predictions = np.loadtxt(exp_dir + '/predictions/predictions_0.csv', delimiter=',')
+    else:
+        cum_predictions = np.loadtxt(exp_dir + '/predictions/cum_predictions.csv', delimiter=',') * (epoch - 1)
+        predictions = np.loadtxt(exp_dir + '/predictions/predictions_' + str(epoch) + '.csv', delimiter=',')
+        cum_predictions = cum_predictions + predictions
+        # remove the prediction file to save storage space
+        os.remove(exp_dir + '/predictions/predictions_' + str(epoch - 1) + '.csv')
+
+    cum_predictions = cum_predictions / (epoch + 1)
+    np.savetxt(exp_dir + '/predictions/cum_predictions.csv', cum_predictions, delimiter=',')
+
+    stats = calculate_stats(cum_predictions, target)
+    return stats
+
+
+
+
+
+
+
+
+
+def zero_shot_eval(model, data, epoch, args):
+    temp_val_a_cls_data = args.val_a_cls_data
+    args.val_a_cls_data = list(data.keys())
+    assert len(args.val_a_cls_data) == 1
+    args.val_a_cls_data = args.val_a_cls_data[0]
+
+    if args.val_a_cls_data not in data:
+        return {}
+    if args.zeroshot_frequency == 0:
+        return {}
+    if (epoch % args.zeroshot_frequency) != 0 and epoch != args.epochs:
+        return {}
+    if args.distributed and not args.horovod:
+        model = model.module
+
+    logging.info(f'Starting zero-shot {args.val_a_cls_data.upper()}.')
+
+    logging.info('Building zero-shot classifier')
+    autocast = get_autocast(args.precision)
+    with autocast():
+        tokenizer = get_tokenizer(HF_HUB_PREFIX+args.model, cache_dir=args.cache_dir)
+        # tokenizer = get_tokenizer("ViT-L-14")
+        classifier = build_zero_shot_classifier(
+            model,
+            tokenizer=tokenizer,
+            classnames=CLASSNAMES[args.val_a_cls_data],
+            templates=OPENAI_IMAGENET_TEMPLATES,
+            num_classes_per_batch=10,
+            device=args.device,
+            use_tqdm=True,
+        )
+
+    logging.info('Using classifier')
+    results = {}
+    if args.val_a_cls_data.lower() == 'audioset':
+        if args.val_a_cls_data in data:
+            stats = validate(model, classifier, data[args.val_a_cls_data].dataloader, args, epoch)
+            results.update(stats)
+    else:
+        if args.val_a_cls_data in data:
+            top1, top5 = run(model, classifier, data[args.val_a_cls_data].dataloader, args)
+            results[f'{args.val_a_cls_data}-zeroshot-val-top1'] = top1
+            results[f'{args.val_a_cls_data}-zeroshot-val-top5'] = top5
+
+    logging.info(f'Finished zero-shot {args.val_a_cls_data.upper()}.')
+
+    args.val_a_cls_data = temp_val_a_cls_data
+    return results
+
+
+
+
+
+
+
+
+
+
+

RAG/Knowledge_Database/languagebind_main/a_cls/zero_shot_classifier.py

@@ -0,0 +1,111 @@
+from functools import partial
+from itertools import islice
+from typing import Callable, List, Optional, Sequence, Union
+
+import torch
+import torch.nn.functional as F
+
+
+def batched(iterable, n):
+    """Batch data into lists of length *n*. The last batch may be shorter.
+    NOTE based on more-itertools impl, to be replaced by python 3.12 itertools.batched impl
+    """
+    it = iter(iterable)
+    while True:
+        batch = list(islice(it, n))
+        if not batch:
+            break
+        yield batch
+
+
+def build_zero_shot_classifier(
+        model,
+        tokenizer,
+        classnames: Sequence[str],
+        templates: Sequence[Union[Callable, str]],
+        num_classes_per_batch: Optional[int] = 10,
+        device: Union[str, torch.device] = 'cpu',
+        use_tqdm: bool = False,
+):
+    """ Build zero-shot classifier weights by iterating over class names in batches
+    Args:
+        model: CLIP model instance
+        tokenizer: CLIP tokenizer instance
+        classnames: A sequence of class (label) names
+        templates: A sequence of callables or format() friendly strings to produce templates per class name
+        num_classes_per_batch: The number of classes to batch together in each forward, all if None
+        device: Device to use.
+        use_tqdm: Enable TQDM progress bar.
+    """
+    assert isinstance(templates, Sequence) and len(templates) > 0
+    assert isinstance(classnames, Sequence) and len(classnames) > 0
+    use_format = isinstance(templates[0], str)
+    num_templates = len(templates)
+    num_classes = len(classnames)
+    if use_tqdm:
+        import tqdm
+        num_iter = 1 if num_classes_per_batch is None else ((num_classes - 1) // num_classes_per_batch + 1)
+        iter_wrap = partial(tqdm.tqdm, total=num_iter, unit_scale=num_classes_per_batch)
+    else:
+        iter_wrap = iter
+
+    def _process_batch(batch_classnames):
+        num_batch_classes = len(batch_classnames)
+        texts = [template.format(c) if use_format else template(c) for c in batch_classnames for template in templates]
+        input_ids, attention_mask = tokenizer(texts)
+        input_ids, attention_mask = input_ids.to(device), attention_mask.to(device)
+        class_embeddings = F.normalize(model.encode_text(input_ids, attention_mask), dim=-1)
+        class_embeddings = class_embeddings.reshape(num_batch_classes, num_templates, -1).mean(dim=1)
+        class_embeddings = class_embeddings / class_embeddings.norm(dim=1, keepdim=True)
+        class_embeddings = class_embeddings.T
+        return class_embeddings
+
+    with torch.no_grad():
+        if num_classes_per_batch:
+            batched_embeds = [_process_batch(batch) for batch in iter_wrap(batched(classnames, num_classes_per_batch))]
+            zeroshot_weights = torch.cat(batched_embeds, dim=1)
+        else:
+            zeroshot_weights = _process_batch(classnames)
+    return zeroshot_weights
+
+
+def build_zero_shot_classifier_legacy(
+        model,
+        tokenizer,
+        classnames: Sequence[str],
+        templates: Sequence[Union[Callable, str]],
+        device: Union[str, torch.device] = 'cpu',
+        use_tqdm: bool = False,
+):
+    """ Build zero-shot classifier weights by iterating over class names 1 by 1
+    Args:
+        model: CLIP model instance
+        tokenizer: CLIP tokenizer instance
+        classnames: A sequence of class (label) names
+        templates: A sequence of callables or format() friendly strings to produce templates per class name
+        device: Device to use.
+        use_tqdm: Enable TQDM progress bar.
+    """
+    assert isinstance(templates, Sequence) and len(templates) > 0
+    assert isinstance(classnames, Sequence) and len(classnames) > 0
+    if use_tqdm:
+        import tqdm
+        iter_wrap = tqdm.tqdm
+    else:
+        iter_wrap = iter
+
+    use_format = isinstance(templates[0], str)
+
+    with torch.no_grad():
+        zeroshot_weights = []
+        for classname in iter_wrap(classnames):
+            texts = [template.format(classname) if use_format else template(classname) for template in templates]
+            texts = tokenizer(texts).to(device)  # tokenize
+            class_embeddings = model.encode_text(texts)
+            class_embedding = F.normalize(class_embeddings, dim=-1).mean(dim=0)
+            class_embedding /= class_embedding.norm()
+            zeroshot_weights.append(class_embedding)
+        zeroshot_weights = torch.stack(zeroshot_weights, dim=1).to(device)
+
+    return zeroshot_weights
+

RAG/Knowledge_Database/languagebind_main/a_cls/zero_shot_metadata.py

@@ -0,0 +1,184 @@
+import os
+
+import pandas as pd
+
+OPENAI_IMAGENET_TEMPLATES = (
+    # lambda c: f'This is a sound of {c}.',
+    lambda c: f'a sound of {c}.',
+)
+# OPENAI_IMAGENET_TEMPLATES = (
+#     lambda c: f'a bad sound of a {c}.',
+#     lambda c: f'a sound of many {c}.',
+#     lambda c: f'a sculpture of a {c}.',
+#     lambda c: f'a sound of the hard to see {c}.',
+#     lambda c: f'a low resolution sound of the {c}.',
+#     lambda c: f'a rendering of a {c}.',
+#     lambda c: f'graffiti of a {c}.',
+#     lambda c: f'a bad sound of the {c}.',
+#     lambda c: f'a cropped sound of the {c}.',
+#     lambda c: f'a tattoo of a {c}.',
+#     lambda c: f'the embroidered {c}.',
+#     lambda c: f'a sound of a hard to see {c}.',
+#     lambda c: f'a bright sound of a {c}.',
+#     lambda c: f'a sound of a clean {c}.',
+#     lambda c: f'a sound of a dirty {c}.',
+#     lambda c: f'a dark sound of the {c}.',
+#     lambda c: f'a drawing of a {c}.',
+#     lambda c: f'a sound of my {c}.',
+#     lambda c: f'the plastic {c}.',
+#     lambda c: f'a sound of the cool {c}.',
+#     lambda c: f'a close-up sound of a {c}.',
+#     lambda c: f'a black and white sound of the {c}.',
+#     lambda c: f'a painting of the {c}.',
+#     lambda c: f'a painting of a {c}.',
+#     lambda c: f'a pixelated sound of the {c}.',
+#     lambda c: f'a sculpture of the {c}.',
+#     lambda c: f'a bright sound of the {c}.',
+#     lambda c: f'a cropped sound of a {c}.',
+#     lambda c: f'a plastic {c}.',
+#     lambda c: f'a sound of the dirty {c}.',
+#     lambda c: f'a jpeg corrupted sound of a {c}.',
+#     lambda c: f'a blurry sound of the {c}.',
+#     lambda c: f'a sound of the {c}.',
+#     lambda c: f'a good sound of the {c}.',
+#     lambda c: f'a rendering of the {c}.',
+#     lambda c: f'a {c} in a video game.',
+#     lambda c: f'a sound of one {c}.',
+#     lambda c: f'a doodle of a {c}.',
+#     lambda c: f'a close-up sound of the {c}.',
+#     lambda c: f'a sound of a {c}.',
+#     lambda c: f'the origami {c}.',
+#     lambda c: f'the {c} in a video game.',
+#     lambda c: f'a sketch of a {c}.',
+#     lambda c: f'a doodle of the {c}.',
+#     lambda c: f'a origami {c}.',
+#     lambda c: f'a low resolution sound of a {c}.',
+#     lambda c: f'the toy {c}.',
+#     lambda c: f'a rendition of the {c}.',
+#     lambda c: f'a sound of the clean {c}.',
+#     lambda c: f'a sound of a large {c}.',
+#     lambda c: f'a rendition of a {c}.',
+#     lambda c: f'a sound of a nice {c}.',
+#     lambda c: f'a sound of a weird {c}.',
+#     lambda c: f'a blurry sound of a {c}.',
+#     lambda c: f'a cartoon {c}.',
+#     lambda c: f'art of a {c}.',
+#     lambda c: f'a sketch of the {c}.',
+#     lambda c: f'a embroidered {c}.',
+#     lambda c: f'a pixelated sound of a {c}.',
+#     lambda c: f'itap of the {c}.',
+#     lambda c: f'a jpeg corrupted sound of the {c}.',
+#     lambda c: f'a good sound of a {c}.',
+#     lambda c: f'a plushie {c}.',
+#     lambda c: f'a sound of the nice {c}.',
+#     lambda c: f'a sound of the small {c}.',
+#     lambda c: f'a sound of the weird {c}.',
+#     lambda c: f'the cartoon {c}.',
+#     lambda c: f'art of the {c}.',
+#     lambda c: f'a drawing of the {c}.',
+#     lambda c: f'a sound of the large {c}.',
+#     lambda c: f'a black and white sound of a {c}.',
+#     lambda c: f'the plushie {c}.',
+#     lambda c: f'a dark sound of a {c}.',
+#     lambda c: f'itap of a {c}.',
+#     lambda c: f'graffiti of the {c}.',
+#     lambda c: f'a toy {c}.',
+#     lambda c: f'itap of my {c}.',
+#     lambda c: f'a sound of a cool {c}.',
+#     lambda c: f'a sound of a small {c}.',
+#     lambda c: f'a tattoo of the {c}.',
+# )
+
+# a much smaller subset of above prompts
+# from https://github.com/openai/CLIP/blob/main/notebooks/Prompt_Engineering_for_ImageNet.ipynb
+SIMPLE_IMAGENET_TEMPLATES = (
+    lambda c: f'itap of a {c}.',
+    lambda c: f'a bad sound of the {c}.',
+    lambda c: f'a origami {c}.',
+    lambda c: f'a sound of the large {c}.',
+    lambda c: f'a {c} in a video game.',
+    lambda c: f'art of the {c}.',
+    lambda c: f'a sound of the small {c}.',
+)
+
+
+PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "class_labels_indices.csv")
+
+
+CLASSNAMES = {
+    'Audioset': tuple(pd.read_csv(PATH).values[:, 2]),
+    'ESC50': (
+        'airplane', 'breathing', 'brushing teeth', 'can opening', 'car horn', 'cat', 'chainsaw', 'chirping birds',
+        'church bells', 'clapping', 'clock alarm', 'clock tick', 'coughing', 'cow', 'crackling fire', 'crickets',
+        'crow', 'crying baby', 'dog', 'door wood creaks', 'door wood knock', 'drinking sipping', 'engine', 'fireworks',
+        'footsteps', 'frog', 'glass breaking', 'hand saw', 'helicopter', 'hen', 'insects', 'keyboard typing',
+        'laughing', 'mouse click', 'pig', 'pouring water', 'rain', 'rooster', 'sea waves', 'sheep', 'siren',
+        'sneezing', 'snoring', 'thunderstorm', 'toilet flush', 'train', 'vacuum cleaner', 'washing machine',
+        'water drops', 'wind'
+    ),
+    'VGGSound': (
+        'air conditioning noise', 'air horn', 'airplane', 'airplane flyby', 'alarm clock ringing',
+        'alligators, crocodiles hissing', 'ambulance siren', 'arc welding', 'baby babbling', 'baby crying',
+        'baby laughter', 'baltimore oriole calling', 'barn swallow calling', 'basketball bounce',
+        'bathroom ventilation fan running', 'beat boxing', 'bee, wasp, etc. buzzing', 'bird chirping, tweeting',
+        'bird squawking', 'bird wings flapping', 'black capped chickadee calling', 'blowtorch igniting',
+        'bouncing on trampoline', 'bowling impact', 'bull bellowing', 'canary calling', 'cap gun shooting',
+        'car engine idling', 'car engine knocking', 'car engine starting', 'car passing by', 'cat caterwauling',
+        'cat growling', 'cat hissing', 'cat meowing', 'cat purring', 'cattle mooing', 'cattle, bovinae cowbell',
+        'cell phone buzzing', 'chainsawing trees', 'cheetah chirrup', 'chicken clucking', 'chicken crowing',
+        'child singing', 'child speech, kid speaking', 'children shouting', 'chimpanzee pant-hooting',
+        'chinchilla barking', 'chipmunk chirping', 'chopping food', 'chopping wood', 'church bell ringing',
+        'civil defense siren', 'cow lowing', 'coyote howling', 'cricket chirping', 'crow cawing', 'cuckoo bird calling',
+        'cupboard opening or closing', 'cutting hair with electric trimmers', 'dinosaurs bellowing', 'disc scratching',
+        'dog barking', 'dog baying', 'dog bow-wow', 'dog growling', 'dog howling', 'dog whimpering',
+        'donkey, ass braying', 'door slamming', 'driving buses', 'driving motorcycle', 'driving snowmobile',
+        'duck quacking', 'eagle screaming', 'eating with cutlery', 'electric grinder grinding',
+        'electric shaver, electric razor shaving', 'elephant trumpeting', 'eletric blender running', 'elk bugling',
+        'engine accelerating, revving, vroom', 'female singing', 'female speech, woman speaking', 'ferret dooking',
+        'fire crackling', 'fire truck siren', 'fireworks banging', 'firing cannon', 'firing muskets',
+        'fly, housefly buzzing', 'foghorn', 'footsteps on snow', 'forging swords', 'fox barking', 'francolin calling',
+        'frog croaking', 'gibbon howling', 'goat bleating', 'golf driving', 'goose honking', 'hail',
+        'hair dryer drying', 'hammering nails', 'heart sounds, heartbeat', 'hedge trimmer running', 'helicopter',
+        'horse clip-clop', 'horse neighing', 'ice cracking', 'ice cream truck, ice cream van', 'lathe spinning',
+        'lawn mowing', 'lighting firecrackers', 'lions growling', 'lions roaring', 'lip smacking',
+        'machine gun shooting', 'magpie calling', 'male singing', 'male speech, man speaking', 'metronome',
+        'missile launch', 'mosquito buzzing', 'motorboat, speedboat acceleration', 'mouse clicking', 'mouse pattering',
+        'mouse squeaking', 'mynah bird singing', 'ocean burbling', 'opening or closing car doors',
+        'opening or closing car electric windows', 'opening or closing drawers', 'orchestra', 'otter growling',
+        'owl hooting', 'parrot talking', 'penguins braying', 'people babbling', 'people battle cry',
+        'people belly laughing', 'people booing', 'people burping', 'people cheering', 'people clapping',
+        'people coughing', 'people crowd', 'people eating', 'people eating apple', 'people eating crisps',
+        'people eating noodle', 'people farting', 'people finger snapping', 'people gargling', 'people giggling',
+        'people hiccup', 'people humming', 'people marching', 'people nose blowing', 'people running',
+        'people screaming', 'people shuffling', 'people slapping', 'people slurping', 'people sneezing',
+        'people sniggering', 'people sobbing', 'people whispering', 'people whistling', 'pheasant crowing',
+        'pig oinking', 'pigeon, dove cooing', 'planing timber', 'plastic bottle crushing', 'playing accordion',
+        'playing acoustic guitar', 'playing badminton', 'playing bagpipes', 'playing banjo', 'playing bass drum',
+        'playing bass guitar', 'playing bassoon', 'playing bongo', 'playing bugle', 'playing castanets',
+        'playing cello', 'playing clarinet', 'playing congas', 'playing cornet', 'playing cymbal', 'playing darts',
+        'playing didgeridoo', 'playing djembe', 'playing double bass', 'playing drum kit', 'playing electric guitar',
+        'playing electronic organ', 'playing erhu', 'playing flute', 'playing french horn', 'playing glockenspiel',
+        'playing gong', 'playing guiro', 'playing hammond organ', 'playing harmonica', 'playing harp',
+        'playing harpsichord', 'playing hockey', 'playing lacrosse', 'playing mandolin', 'playing marimba, xylophone',
+        'playing oboe', 'playing piano', 'playing saxophone', 'playing shofar', 'playing sitar', 'playing snare drum',
+        'playing squash', 'playing steel guitar, slide guitar', 'playing steelpan', 'playing synthesizer',
+        'playing tabla', 'playing table tennis', 'playing tambourine', 'playing tennis', 'playing theremin',
+        'playing timbales', 'playing timpani', 'playing trombone', 'playing trumpet', 'playing tuning fork',
+        'playing tympani', 'playing ukulele', 'playing vibraphone', 'playing violin, fiddle', 'playing volleyball',
+        'playing washboard', 'playing zither', 'police car (siren)', 'police radio chatter', 'popping popcorn',
+        'printer printing', 'pumping water', 'race car, auto racing', 'railroad car, train wagon', 'raining', 'rapping',
+        'reversing beeps', 'ripping paper', 'roller coaster running', 'rope skipping', 'rowboat, canoe, kayak rowing',
+        'running electric fan', 'sailing', 'scuba diving', 'sea lion barking', 'sea waves', 'sharpen knife',
+        'sheep bleating', 'shot football', 'singing bowl', 'singing choir', 'skateboarding', 'skidding', 'skiing',
+        'sliding door', 'sloshing water', 'slot machine', 'smoke detector beeping', 'snake hissing', 'snake rattling',
+        'splashing water', 'spraying water', 'squishing water', 'stream burbling', 'strike lighter', 'striking bowling',
+        'striking pool', 'subway, metro, underground', 'swimming', 'tap dancing', 'tapping guitar',
+        'telephone bell ringing', 'thunder', 'toilet flushing', 'tornado roaring', 'tractor digging', 'train horning',
+        'train wheels squealing', 'train whistling', 'turkey gobbling', 'typing on computer keyboard',
+        'typing on typewriter', 'underwater bubbling', 'using sewing machines', 'vacuum cleaner cleaning floors',
+        'vehicle horn, car horn, honking', 'volcano explosion', 'warbler chirping', 'waterfall burbling',
+        'whale calling', 'wind chime', 'wind noise', 'wind rustling leaves', 'wood thrush calling',
+        'woodpecker pecking tree', 'writing on blackboard with chalk', 'yodelling', 'zebra braying'
+    )
+
+}

RAG/Knowledge_Database/languagebind_main/a_cls/zeroshot_cls.py

@@ -0,0 +1,46 @@
+
+import json
+import logging
+import os
+from training.distributed import is_master
+from .zero_shot import zero_shot_eval
+
+try:
+    import wandb
+except ImportError:
+    wandb = None
+
+
+
+def evaluate_a_cls(model, data, epoch, args, tb_writer=None):
+    metrics = {}
+    if not is_master(args):
+        return metrics
+    model.eval()
+
+    zero_shot_metrics = zero_shot_eval(model, data, epoch, args)
+    metrics.update(zero_shot_metrics)
+
+    if not metrics:
+        return metrics
+
+    logging.info(
+        f"Eval Epoch: {epoch} "
+        + "\t".join([f"{k}: {round(v, 4):.4f}" for k, v in metrics.items()])
+    )
+    if args.save_logs:
+        for name, val in metrics.items():
+            if tb_writer is not None:
+                tb_writer.add_scalar(f"val/a_cls/{args.val_a_cls_data[0].lower()}/{name}", val, epoch)
+        args.a_cls_output_dir = os.path.join(args.log_base_path, f'a_cls/{args.val_a_cls_data[0].lower()}')
+        os.makedirs(args.a_cls_output_dir, exist_ok=True)
+        with open(os.path.join(args.a_cls_output_dir, "results.jsonl"), "a+") as f:
+            f.write(json.dumps(metrics))
+            f.write("\n")
+
+    if args.wandb:
+        assert wandb is not None, 'Please install wandb.'
+        for name, val in metrics.items():
+            wandb.log({f"val/{name}": val, 'epoch': epoch})
+
+    return metrics

RAG/Knowledge_Database/languagebind_main/al_ret/data_dataloaders.py

@@ -0,0 +1,28 @@
+import argparse
+import torch
+from torch.utils.data import DataLoader
+
+from data.build_datasets import get_data
+from data.process_audio import get_audio_transform
+from .dataloader_msrvtt_retrieval import MSRVTT_DataLoader
+
+def dataloader_msrvtt_test(args, tokenizer, subset="test"):
+    msrvtt_testset = MSRVTT_DataLoader(
+        csv_path=args.val_csv,
+        features_path=args.features_path,
+        max_words=args.max_words,
+        tokenizer=tokenizer,
+        transform=get_audio_transform(args)
+    )
+    dataloader_msrvtt = DataLoader(
+        msrvtt_testset,
+        batch_size=args.batch_size_val,
+        num_workers=args.num_thread_reader,
+        shuffle=False,
+        drop_last=False,
+    )
+    return dataloader_msrvtt, len(msrvtt_testset)
+
+
+DATALOADER_DICT = {}
+DATALOADER_DICT["msrvtt"] = {"val":dataloader_msrvtt_test, "test":None}

RAG/Knowledge_Database/languagebind_main/al_ret/dataloader_msrvtt_retrieval.py

@@ -0,0 +1,114 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+
+import os
+
+import torchaudio
+from torch.utils.data import Dataset
+import numpy as np
+import pandas as pd
+from collections import defaultdict
+import json
+import random
+
+from torchvision.io import read_video
+
+
+class MSRVTT_DataLoader(Dataset):
+    """MSRVTT dataset loader."""
+    def __init__(
+            self,
+            csv_path,
+            features_path,
+            tokenizer,
+            transform=77,
+            max_words=30,
+    ):
+        self.data = pd.read_csv(csv_path)
+        self.features_path = features_path
+        self.max_words = max_words
+        self.tokenizer = tokenizer
+
+        # self.rawVideoExtractor = RawVideoExtractor(framerate=feature_framerate, size=image_resolution)
+        self.transform = transform
+        self.SPECIAL_TOKEN = {"CLS_TOKEN": "<|startoftext|>", "SEP_TOKEN": "<|endoftext|>",
+                              "MASK_TOKEN": "[MASK]", "UNK_TOKEN": "[UNK]", "PAD_TOKEN": "[PAD]"}
+
+
+
+    def __len__(self):
+        return len(self.data)
+
+    def _get_text(self, video_id, sentence):
+        choice_video_ids = [video_id]
+        n_caption = len(choice_video_ids)
+
+        k = n_caption
+        pairs_text = np.zeros((k, self.max_words), dtype=np.long)
+        pairs_mask = np.zeros((k, self.max_words), dtype=np.long)
+        pairs_segment = np.zeros((k, self.max_words), dtype=np.long)
+
+        for i, video_id in enumerate(choice_video_ids):
+            # words = self.tokenizer.tokenize(sentence)
+            #
+            # words = [self.SPECIAL_TOKEN["CLS_TOKEN"]] + words
+            # total_length_with_CLS = self.max_words - 1
+            # if len(words) > total_length_with_CLS:
+            #     words = words[:total_length_with_CLS]
+            # words = words + [self.SPECIAL_TOKEN["SEP_TOKEN"]]
+            #
+            # input_ids = self.tokenizer.convert_tokens_to_ids(words)
+            # input_mask = [1] * len(input_ids)
+            # segment_ids = [0] * len(input_ids)
+
+
+            output = self.tokenizer(sentence)
+
+            input_ids = output[0].squeeze()
+            input_mask = output[1].squeeze()
+            segment_ids = [0] * len(input_ids)
+
+
+            while len(input_ids) < self.max_words:
+                input_ids.append(0)
+                input_mask.append(0)
+                segment_ids.append(0)
+            assert len(input_ids) == self.max_words
+            assert len(input_mask) == self.max_words
+            assert len(segment_ids) == self.max_words
+
+            pairs_text[i] = np.array(input_ids)
+            pairs_mask[i] = np.array(input_mask)
+            pairs_segment[i] = np.array(segment_ids)
+
+        return pairs_text, pairs_mask, pairs_segment, choice_video_ids
+
+    def _get_rawvideo(self, choice_video_ids):
+        # Pair x L x T x 3 x H x W
+        audio = np.zeros((len(choice_video_ids), 3,
+                          self.transform.num_mel_bins, self.transform.target_length), dtype=np.float)
+        assert len(choice_video_ids) == 1
+        for i, video_id in enumerate(choice_video_ids):
+            # Individual for YoucokII dataset, due to it video format
+            video_path = os.path.join(self.features_path, "{}.mp4".format(video_id))
+            if os.path.exists(video_path) is False:
+                video_path = video_path.replace(".mp4", ".webm")
+
+            # raw_video_data = self.rawVideoExtractor.get_video_data(video_path)
+            # _, raw_audio_data, info = read_video(video_path, pts_unit='sec')
+            # audio_data = self.transform((raw_audio_data, info['audio_fps']))
+
+            audio_data = torchaudio.load(video_path.replace('mp4', 'wav'))
+            audio_data = self.transform(audio_data)
+            # audio[i] = audio_data
+        return audio_data
+
+    def __getitem__(self, idx):
+        video_id = self.data['video_id'].values[idx]
+        sentence = self.data['sentence'].values[idx]
+
+        pairs_text, pairs_mask, pairs_segment, choice_video_ids = self._get_text(video_id, sentence)
+        audio_data = self._get_rawvideo(choice_video_ids)
+        return audio_data, pairs_text, pairs_mask

RAG/Knowledge_Database/languagebind_main/al_ret/datasets.py

@@ -0,0 +1,137 @@
+import logging
+import os.path
+import random
+
+import numpy as np
+import pandas as pd
+import torch
+from torch.utils.data import Dataset
+from data.build_datasets import DataInfo
+from open_clip import get_input_dtype, get_tokenizer
+from open_clip.factory import HF_HUB_PREFIX
+from data.process_audio import get_audio_transform, torchaudio_loader
+
+class Audiocaps_dataset(Dataset):
+    def __init__(self, data_path, transform, loader, tokenizer):
+        super(Audiocaps_dataset, self).__init__()
+        self.audio_root = data_path
+        raw_meta = pd.read_csv(f'{self.audio_root}/audiocaps_test.tsv', delimiter='\t').values
+        audio_ids = list(set(raw_meta[:, 1].tolist()))
+        captions = {}
+        for i in raw_meta:
+            if captions.get(i[1], None) is None:
+                captions[i[1]] = [i[2]]
+            else:
+                captions[i[1]] = captions[i[1]] + [i[2]]
+        # captions = {i[:1][0]: i[1:].tolist() for i in raw_meta}
+
+
+        self.sample_len = 0
+        self.sentences_dict = {}
+        self.cut_off_points = []
+        for audio_id in audio_ids:
+            assert audio_id in captions
+            for cap in captions[audio_id]:
+                cap_txt = cap
+                self.sentences_dict[len(self.sentences_dict)] = (audio_id[10:], cap_txt)
+            self.cut_off_points.append(len(self.sentences_dict))
+
+        self.multi_sentence_per_audio = True  # !!! important tag for eval
+        if self.multi_sentence_per_audio:
+            # if self.subset == "val" or self.subset == "test":
+            self.sentence_num = len(self.sentences_dict)
+            self.audio_num = len(audio_ids)
+            assert len(self.cut_off_points) == self.audio_num
+            print("Sentence number: {}".format(self.sentence_num))
+            print("Video number: {}".format(self.audio_num))
+
+        self.sample_len = len(self.sentences_dict)
+
+        self.transform = transform
+        self.torchaudio_loader = loader
+        self.tokenizer = tokenizer
+
+    def __len__(self):
+        return self.sample_len
+
+    def __getitem__(self, idx):
+        audiocap_id, caption = self.sentences_dict[idx]
+
+        audio_path = os.path.join(self.audio_root, audiocap_id)
+        audio = self.torchaudio_loader(audio_path)
+        audio_data = self.transform(audio)
+
+        input_ids, attention_mask = self.tokenizer(caption)
+        return audio_data, input_ids.squeeze(), attention_mask.squeeze()
+
+
+class Clotho_dataset(Dataset):
+    def __init__(self, data_path, transform, loader, tokenizer):
+        super(Clotho_dataset, self).__init__()
+        self.audio_root = data_path
+        raw_meta = pd.read_csv(f'{self.audio_root}/CLOTHO_retrieval_dataset/clotho_captions_evaluation.csv').values
+        audio_ids = raw_meta[:, 0].tolist()
+        captions = {i[:1][0]: i[1:].tolist() for i in raw_meta}
+        # self.meta = pd.DataFrame(np.vstack([np.vstack([raw_meta[:, 0], raw_meta[:, i]]).T for i in range(1, 6)]),
+        #                          columns=['uniq_id', 'text'])
+
+        self.sample_len = 0
+        self.sentences_dict = {}
+        self.cut_off_points = []
+        for audio_id in audio_ids:
+            assert audio_id in captions
+            for cap in captions[audio_id]:
+                cap_txt = cap
+                self.sentences_dict[len(self.sentences_dict)] = (audio_id, cap_txt)
+            self.cut_off_points.append(len(self.sentences_dict))
+
+        self.multi_sentence_per_audio = True    # !!! important tag for eval
+        if self.multi_sentence_per_audio:
+            # if self.subset == "val" or self.subset == "test":
+            self.sentence_num = len(self.sentences_dict)
+            self.audio_num = len(audio_ids)
+            assert len(self.cut_off_points) == self.audio_num
+            print("Sentence number: {}".format(self.sentence_num))
+            print("Video number: {}".format(self.audio_num))
+
+        self.sample_len = len(self.sentences_dict)
+
+        self.transform = transform
+        self.torchaudio_loader = loader
+        self.tokenizer = tokenizer
+
+    def __len__(self):
+        return self.sample_len
+
+    def __getitem__(self, idx):
+        audiocap_id, caption = self.sentences_dict[idx]
+        # audiocap_id = self.meta['uniq_id'][idx]
+        audio_path = os.path.join(self.audio_root, f'evaluation/{audiocap_id}')
+        audio = self.torchaudio_loader(audio_path)
+        audio_data = self.transform(audio)
+
+        # caption = self.meta['text'][idx]
+        input_ids, attention_mask = self.tokenizer(caption)
+        return audio_data, input_ids.squeeze(), attention_mask.squeeze()
+
+def get_audio_dataset(args):
+    data_path = args.audio_data_path
+    transform = get_audio_transform(args)
+    tokenizer = get_tokenizer(HF_HUB_PREFIX+args.model, cache_dir=args.cache_dir)
+
+    if args.val_al_ret_data.lower() == 'audiocaps':
+        dataset = Audiocaps_dataset(data_path, transform=transform, loader=torchaudio_loader, tokenizer=tokenizer)
+    elif args.val_al_ret_data.lower() == 'clotho':
+        dataset = Clotho_dataset(data_path, transform=transform, loader=torchaudio_loader, tokenizer=tokenizer)
+    else:
+        raise ValueError(f'unsupport dataset {args.val_al_ret_data}')
+
+    dataloader = torch.utils.data.DataLoader(
+        dataset,
+        batch_size=args.batch_size,
+        num_workers=args.workers,
+        shuffle=False,
+        drop_last=False,
+    )
+
+    return dataloader

RAG/Knowledge_Database/languagebind_main/al_ret/metrics.py

@@ -0,0 +1,70 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+from __future__ import print_function
+
+import numpy as np
+import torch
+
+def compute_metrics(x):
+    sx = np.sort(-x, axis=1)
+    d = np.diag(-x)
+    d = d[:, np.newaxis]
+    ind = sx - d
+    ind = np.where(ind == 0)
+    ind = ind[1]
+    metrics = {}
+    metrics['R1'] = float(np.sum(ind == 0)) * 100 / len(ind)
+    metrics['R5'] = float(np.sum(ind < 5)) * 100 / len(ind)
+    metrics['R10'] = float(np.sum(ind < 10)) * 100 / len(ind)
+    metrics['MR'] = np.median(ind) + 1
+    metrics["MedianR"] = metrics['MR']
+    metrics["MeanR"] = np.mean(ind) + 1
+    # metrics["cols"] = [int(i) for i in list(ind)]
+    return metrics
+
+def print_computed_metrics(metrics):
+    r1 = metrics['R1']
+    r5 = metrics['R5']
+    r10 = metrics['R10']
+    mr = metrics['MR']
+    print('R@1: {:.4f} - R@5: {:.4f} - R@10: {:.4f} - Median R: {}'.format(r1, r5, r10, mr))
+
+# below two functions directly come from: https://github.com/Deferf/Experiments
+def tensor_text_to_video_metrics(sim_tensor, top_k = [1,5,10]):
+    if not torch.is_tensor(sim_tensor):
+      sim_tensor = torch.tensor(sim_tensor)
+
+    # Permute sim_tensor so it represents a sequence of text-video similarity matrices.
+    # Then obtain the double argsort to position the rank on the diagonal
+    stacked_sim_matrices = sim_tensor.permute(1, 0, 2)
+    first_argsort = torch.argsort(stacked_sim_matrices, dim = -1, descending= True)
+    second_argsort = torch.argsort(first_argsort, dim = -1, descending= False)
+
+    # Extracts ranks i.e diagonals
+    ranks = torch.flatten(torch.diagonal(second_argsort, dim1 = 1, dim2 = 2))
+
+    # Now we need to extract valid ranks, as some belong to inf padding values
+    permuted_original_data = torch.flatten(torch.diagonal(sim_tensor, dim1 = 0, dim2 = 2))
+    mask = ~ torch.logical_or(torch.isinf(permuted_original_data), torch.isnan(permuted_original_data))
+    valid_ranks = ranks[mask]
+    # A quick dimension check validates our results, there may be other correctness tests pending
+    # Such as dot product localization, but that is for other time.
+    #assert int(valid_ranks.shape[0]) ==  sum([len(text_dict[k]) for k in text_dict])
+    if not torch.is_tensor(valid_ranks):
+      valid_ranks = torch.tensor(valid_ranks)
+    results = {f"R{k}": float(torch.sum(valid_ranks < k) * 100 / len(valid_ranks)) for k in top_k}
+    results["MedianR"] = float(torch.median(valid_ranks + 1))
+    results["MeanR"] = float(np.mean(valid_ranks.numpy() + 1))
+    results["Std_Rank"] = float(np.std(valid_ranks.numpy() + 1))
+    results['MR'] = results["MedianR"]
+    return results
+
+def tensor_video_to_text_sim(sim_tensor):
+    if not torch.is_tensor(sim_tensor):
+      sim_tensor = torch.tensor(sim_tensor)
+    # Code to avoid nans
+    sim_tensor[sim_tensor != sim_tensor] = float('-inf')
+    # Forms a similarity matrix for use with rank at k
+    values, _ = torch.max(sim_tensor, dim=1, keepdim=True)
+    return torch.squeeze(values).T

RAG/Knowledge_Database/languagebind_main/al_ret/precision.py

@@ -0,0 +1,12 @@
+import torch
+from contextlib import suppress
+
+
+def get_autocast(precision):
+    if precision == 'amp':
+        return torch.cuda.amp.autocast
+    elif precision == 'amp_bfloat16' or precision == 'amp_bf16':
+        # amp_bfloat16 is more stable than amp float16 for clip training
+        return lambda: torch.cuda.amp.autocast(dtype=torch.bfloat16)
+    else:
+        return suppress

RAG/Knowledge_Database/languagebind_main/al_ret/retrieval.py

@@ -0,0 +1,266 @@
+
+import json
+import logging
+import os
+import numpy as np
+import torch
+
+from training.distributed import is_master
+from .zero_shot import zero_shot_eval
+from .util import parallel_apply
+from .metrics import compute_metrics, tensor_text_to_video_metrics, tensor_video_to_text_sim
+from torch.nn import functional as F
+try:
+    import wandb
+except ImportError:
+    wandb = None
+
+
+#
+# def evaluate_al_ret(model, data, epoch, args, tb_writer=None):
+#     metrics = {}
+#     if not is_master(args):
+#         return metrics
+#     model.eval()
+#
+#     zero_shot_metrics = zero_shot_eval(model, data, epoch, args)
+#     metrics.update(zero_shot_metrics)
+#
+#     if not metrics:
+#         return metrics
+#
+#     logging.info(
+#         f"Eval Epoch: {epoch} "
+#         + "\t".join([f"{k}: {round(v, 4):.4f}" for k, v in metrics.items()])
+#     )
+#
+#     if args.save_logs:
+#         for name, val in metrics.items():
+#             if tb_writer is not None:
+#                 tb_writer.add_scalar(f"val/al_ret/{name}", val, epoch)
+#         args.al_ret_output_dir = os.path.join(args.log_base_path, 'al_ret')
+#         os.makedirs(args.al_ret_output_dir, exist_ok=True)
+#         with open(os.path.join(args.al_ret_output_dir, "results.jsonl"), "a+") as f:
+#             f.write(json.dumps(metrics))
+#             f.write("\n")
+#
+#     if args.wandb:
+#         assert wandb is not None, 'Please install wandb.'
+#         for name, val in metrics.items():
+#             wandb.log({f"val/{name}": val, 'epoch': epoch})
+#
+#     return metrics
+
+
+
+def _run_on_single_gpu(model,
+                       # batch_list_t, batch_list_v,
+                       batch_sequence_output_list, batch_visual_output_list):
+    sim_matrix = []
+    for idx1 in range(len(batch_sequence_output_list)):
+        # input_mask, segment_ids, *_tmp = b1
+        sequence_output = batch_sequence_output_list[idx1]
+        each_row = []
+        for idx2 in range(len(batch_visual_output_list)):
+            # video_mask, *_tmp = b2
+            visual_output = batch_visual_output_list[idx2]
+            # b1b2_logits, *_tmp = model.get_similarity_logits(sequence_output, visual_output, input_mask, video_mask,
+            #                                                          loose_type=model.loose_type)
+            # logging.info(f"{model.logit_scale.device}, {visual_output.device}, {sequence_output.device}")
+            b1b2_logits = model.logit_scale * sequence_output @ visual_output.T
+            # print(model.logit_scale.device, visual_output.device, sequence_output.device)
+            # logging.info(f"{b1b2_logits.shape}, {b1b2_logits.device}")
+            b1b2_logits = b1b2_logits.cpu().detach().numpy()
+            each_row.append(b1b2_logits)
+        each_row = np.concatenate(tuple(each_row), axis=-1)
+        sim_matrix.append(each_row)
+    return sim_matrix
+
+def evaluate_al_ret(model, data, epoch, args, tb_writer=None):
+    if is_master(args) and (args.val_frequency and ((epoch % args.val_frequency) == 0 or epoch == args.epochs)):
+        # print(data)
+        val_al_ret_data = list(data.keys())
+        # print(val_vl_ret_data)
+        assert len(val_al_ret_data) == 1
+        val_al_ret_data = val_al_ret_data[0]
+        test_dataloader = data[val_al_ret_data]
+        # print(len(test_dataloader))
+        # print(len(test_dataloader))
+        # print(len(test_dataloader))
+        # print(len(test_dataloader))
+        device = model.device
+        n_gpu = torch.cuda.device_count()
+        logging.info(f"\nEval Epoch: {epoch}, eval Audio-Text Retrieval under {val_al_ret_data.upper()} test data")
+        if hasattr(model, 'module'):
+            model = model.module.to(device)
+        else:
+            model = model.to(device)
+        # #################################################################
+        ## below variables are used to multi-sentences retrieval
+        # multi_sentence_: important tag for eval
+        # cut_off_points: used to tag the label when calculate the metric
+        # sentence_num: used to cut the sentence representation
+        # video_num: used to cut the video representation
+        # #################################################################
+        multi_sentence_ = False
+        cut_off_points_, sentence_num_, video_num_ = [], -1, -1
+        if hasattr(test_dataloader.dataset, 'multi_sentence_per_audio') and test_dataloader.dataset.multi_sentence_per_audio:
+        # if False:
+            multi_sentence_ = True
+            cut_off_points_ = test_dataloader.dataset.cut_off_points
+            sentence_num_ = test_dataloader.dataset.sentence_num
+            video_num_ = test_dataloader.dataset.audio_num
+            cut_off_points_ = [itm - 1 for itm in cut_off_points_]
+
+        if multi_sentence_:
+            print("Eval under the multi-sentence per audio clip setting.")
+            print("sentence num: {}, video num: {}".format(sentence_num_, video_num_))
+            logging.info("Eval under the multi-sentence per audio clip setting.")
+            logging.info("sentence num: {}, video num: {}".format(sentence_num_, video_num_))
+
+        model.eval()
+        with torch.no_grad():
+            # batch_list_t = []
+            # batch_list_v = []
+            batch_sequence_output_list, batch_visual_output_list = [], []
+            total_video_num = 0
+
+            # ----------------------------
+            # 1. cache the features
+            # ----------------------------
+            for bid, batch in enumerate(test_dataloader):
+                # batch = tuple(t.to(device) for t in batch)
+                video, input_ids, attention_mask = batch
+                # print(input_ids.shape, video.shape, video.dtype)
+                input_ids = input_ids.squeeze().to(device)
+                attention_mask = attention_mask.squeeze().to(device)
+                # video = video.squeeze().permute(0, 2, 1, 3, 4).float().to(device)
+                video = video.float().to(device)
+
+
+
+                # print(input_ids.shape, video.shape, video.dtype)
+                # print(input_ids.shape, video.shape)
+                if multi_sentence_:
+                    # multi-sentences retrieval means: one clip has two or more descriptions.
+                    b, *_t = video.shape
+                    sequence_output = model.encode_text(input_ids, attention_mask)
+                    # logging.info(f'multi: {sequence_output.shape}')
+                    # sequence_output = model.get_sequence_output(input_ids, segment_ids, input_mask)
+                    batch_sequence_output_list.append(sequence_output)
+                    # batch_list_t.append((input_mask, segment_ids,))
+                    # 0 16
+                    s_, e_ = total_video_num, total_video_num + b
+                    filter_inds = [itm - s_ for itm in cut_off_points_ if itm >= s_ and itm < e_] # cut_off_points_ [0 4 9 14]
+
+                    if len(filter_inds) > 0:
+                        # video, video_mask = video[filter_inds, ...], video_mask[filter_inds, ...]
+                        # print('before', video.shape)
+                        video = video[filter_inds, ...]
+                        # print('after', video.shape)
+                        # visual_output = model.get_visual_output(video, video_mask)
+                        visual_output = model.encode_image(video)
+                        batch_visual_output_list.append(visual_output)
+                        # batch_list_v.append((video_mask,))
+                    total_video_num += b
+                else:
+                    sequence_output = model.encode_text(input_ids, attention_mask)
+                    visual_output = model.encode_image(video)
+                    # sequence_output, visual_output = model.get_sequence_visual_output(input_ids, segment_ids, input_mask, video, video_mask)
+
+                    batch_sequence_output_list.append(sequence_output)
+                    # batch_list_t.append((input_mask, segment_ids,))
+
+                    batch_visual_output_list.append(visual_output)
+                    # batch_list_v.append((video_mask,))
+
+                print(f"Process {val_al_ret_data.upper()}: {bid}/{len(test_dataloader)}\r", end='')
+            # ----------------------------------
+            # 2. calculate the similarity
+            # ----------------------------------
+            n_gpu = torch.cuda.device_count()
+            if n_gpu > 1:
+                # print('n_gpu > 1')
+                device_ids = list(range(n_gpu))
+                # print('device_ids', device_ids)
+                batch_t_output_splits = []
+                batch_v_output_splits = []
+                bacth_len = len(batch_sequence_output_list)
+                # print(bacth_len)
+                split_len = (bacth_len + n_gpu - 1) // n_gpu
+                for dev_id in device_ids:
+                    s_, e_ = dev_id * split_len, (dev_id + 1) * split_len
+                    if dev_id == 0:
+
+                        batch_t_output_splits.append(batch_sequence_output_list[s_:e_])
+                        batch_v_output_splits.append(batch_visual_output_list)
+                        # print(len(batch_sequence_output_list[s_:e_]), len(batch_visual_output_list))
+                    else:
+                        devc = torch.device('cuda:{}'.format(str(dev_id)))
+
+                        devc_batch_list = [b.to(devc) for b in batch_sequence_output_list[s_:e_]]
+                        batch_t_output_splits.append(devc_batch_list)
+                        devc_batch_list = [b.to(devc) for b in batch_visual_output_list]
+                        batch_v_output_splits.append(devc_batch_list)
+                        # print(len(devc_batch_list), len(devc_batch_list))
+                parameters_tuple_list = [(
+                                          batch_t_output_splits[dev_id], batch_v_output_splits[dev_id]) for dev_id in device_ids]
+                parallel_outputs = parallel_apply(_run_on_single_gpu, model, parameters_tuple_list, device_ids)
+                sim_matrix = []
+                for idx in range(len(parallel_outputs)):
+                    sim_matrix += parallel_outputs[idx]
+                sim_matrix = np.concatenate(tuple(sim_matrix), axis=0)
+            else:
+                sim_matrix = _run_on_single_gpu(model,
+                                                # batch_list_t, batch_list_v,
+                                                batch_sequence_output_list, batch_visual_output_list)
+                sim_matrix = np.concatenate(tuple(sim_matrix), axis=0)
+        #####################################################################
+        if multi_sentence_:
+
+            logging.info(f"{val_al_ret_data.upper()} before reshape, sim matrix size: {sim_matrix.shape}")
+            cut_off_points2len_ = [itm + 1 for itm in cut_off_points_]
+            max_length = max([e_-s_ for s_, e_ in zip([0]+cut_off_points2len_[:-1], cut_off_points2len_)])
+            sim_matrix_new = []
+            for s_, e_ in zip([0] + cut_off_points2len_[:-1], cut_off_points2len_):
+                sim_matrix_new.append(np.concatenate((sim_matrix[s_:e_],
+                                                      np.full((max_length-e_+s_, sim_matrix.shape[1]), -np.inf)), axis=0))
+            sim_matrix = np.stack(tuple(sim_matrix_new), axis=0)
+            logging.info(f"{val_al_ret_data.upper()} after reshape, sim matrix size: {sim_matrix.shape}")
+
+            tv_metrics = tensor_text_to_video_metrics(sim_matrix)
+            # vt_metrics = compute_metrics(tensor_video_to_text_sim(sim_matrix))
+        else:
+            logging.info(f"{val_al_ret_data.upper()} sim matrix size: {sim_matrix.shape[0]}, {sim_matrix.shape[1]}")
+            t2v_sim_matrix = torch.from_numpy(sim_matrix).cuda()
+            # t2v_sim_matrix = t2v_sim_matrix * F.softmax(t2v_sim_matrix*10, dim=0) * len(t2v_sim_matrix)
+            tv_metrics = compute_metrics(t2v_sim_matrix.cpu().numpy())
+
+
+            # vt_metrics = compute_metrics(t2v_sim_matrix.T.cpu().numpy())
+
+            logging.info('\t Length-T: {}, Length-V:{}'.format(len(sim_matrix), len(sim_matrix[0])))
+
+        logging.info(f"{val_al_ret_data.upper()} Text-to-Audio:")
+        logging.info('\t>>>  R@1: {:.1f} - R@5: {:.1f} - R@10: {:.1f} - Median R: {:.1f} - Mean R: {:.1f}'.
+                    format(tv_metrics['R1'], tv_metrics['R5'], tv_metrics['R10'], tv_metrics['MR'], tv_metrics['MeanR']))
+        # logging.info(f"{val_al_ret_data.upper()} Text-to-Audio:")
+        # logging.info('\t>>>  V2T$R@1: {:.1f} - V2T$R@5: {:.1f} - V2T$R@10: {:.1f} - V2T$Median R: {:.1f} - V2T$Mean R: {:.1f}'.
+        #             format(vt_metrics['R1'], vt_metrics['R5'], vt_metrics['R10'], vt_metrics['MR'], vt_metrics['MeanR']))
+
+
+        if args.save_logs:
+            for name, val in tv_metrics.items():
+                if tb_writer is not None:
+                    tb_writer.add_scalar(f"val/al_ret/{val_al_ret_data}/t2a/{name}", val, epoch)
+            # for name, val in vt_metrics.items():
+            #     if tb_writer is not None:
+            #         tb_writer.add_scalar(f"val/al_ret/{val_al_ret_data}/v2t/{name}", val, epoch)
+
+            args.al_ret_output_dir = os.path.join(args.log_base_path, f'al_ret/{val_al_ret_data}')
+            os.makedirs(args.al_ret_output_dir, exist_ok=True)
+            with open(os.path.join(args.al_ret_output_dir, "results.jsonl"), "a+") as f:
+                f.write(json.dumps({'t2a': tv_metrics}))
+                f.write("\n")
+                # f.write(json.dumps({'v2t': vt_metrics}))
+                # f.write("\n")

RAG/Knowledge_Database/languagebind_main/al_ret/util.py

@@ -0,0 +1,73 @@
+import torch
+import torch.nn as nn
+import threading
+from torch._utils import ExceptionWrapper
+import logging
+
+def get_a_var(obj):
+    if isinstance(obj, torch.Tensor):
+        return obj
+
+    if isinstance(obj, list) or isinstance(obj, tuple):
+        for result in map(get_a_var, obj):
+            if isinstance(result, torch.Tensor):
+                return result
+    if isinstance(obj, dict):
+        for result in map(get_a_var, obj.items()):
+            if isinstance(result, torch.Tensor):
+                return result
+    return None
+
+def parallel_apply(fct, model, inputs, device_ids):
+    modules = nn.parallel.replicate(model, device_ids)
+    assert len(modules) == len(inputs)
+    lock = threading.Lock()
+    results = {}
+    grad_enabled = torch.is_grad_enabled()
+
+    def _worker(i, module, input):
+        torch.set_grad_enabled(grad_enabled)
+        device = get_a_var(input).get_device()
+        try:
+            with torch.cuda.device(device):
+                # this also avoids accidental slicing of `input` if it is a Tensor
+                if not isinstance(input, (list, tuple)):
+                    input = (input,)
+                output = fct(module, *input)
+            with lock:
+                results[i] = output
+        except Exception:
+            with lock:
+                results[i] = ExceptionWrapper(where="in replica {} on device {}".format(i, device))
+
+    if len(modules) > 1:
+        threads = [threading.Thread(target=_worker, args=(i, module, input))
+                   for i, (module, input) in enumerate(zip(modules, inputs))]
+
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+    else:
+        _worker(0, modules[0], inputs[0])
+
+    outputs = []
+    for i in range(len(inputs)):
+        output = results[i]
+        if isinstance(output, ExceptionWrapper):
+            output.reraise()
+        outputs.append(output)
+    return outputs
+
+def get_logger(filename=None):
+    logger = logging.getLogger('logger')
+    logger.setLevel(logging.DEBUG)
+    logging.basicConfig(format='%(asctime)s - %(levelname)s -   %(message)s',
+                    datefmt='%m/%d/%Y %H:%M:%S',
+                    level=logging.INFO)
+    if filename is not None:
+        handler = logging.FileHandler(filename)
+        handler.setLevel(logging.DEBUG)
+        handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s: %(message)s'))
+        logging.getLogger().addHandler(handler)
+    return logger

RAG/Knowledge_Database/languagebind_main/al_ret/zero_shot.py

@@ -0,0 +1,91 @@
+import logging
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from tqdm import tqdm
+
+from open_clip import get_input_dtype, get_tokenizer
+from open_clip.factory import HF_HUB_PREFIX
+from .precision import get_autocast
+
+def compute_metrics(x):
+    sx = np.sort(-x, axis=1)
+    d = np.diag(-x)
+    d = d[:, np.newaxis]
+    ind = sx - d
+    ind = np.where(ind == 0)
+    ind = ind[1]
+    metrics = {}
+    metrics['R1'] = float(np.sum(ind == 0)) * 100 / len(ind)
+    metrics['R5'] = float(np.sum(ind < 5)) * 100 / len(ind)
+    metrics['R10'] = float(np.sum(ind < 10)) * 100 / len(ind)
+    metrics['MR'] = np.median(ind) + 1
+    metrics["MedianR"] = metrics['MR']
+    metrics["MeanR"] = np.mean(ind) + 1
+    # metrics["cols"] = [int(i) for i in list(ind)]
+    return metrics
+
+
+def _run_on_single_gpu(model, batch_sequence_output_list, batch_visual_output_list):
+    sim_matrix = []
+    logit_scale = model.logit_scale.exp()
+    for idx1, sequence_output in enumerate(batch_sequence_output_list):
+        each_row = []
+        for idx2, visual_output in enumerate(batch_visual_output_list):
+            b1b2_logits = logit_scale * torch.matmul(sequence_output, visual_output.t())
+            b1b2_logits = b1b2_logits.cpu().detach().numpy()
+            each_row.append(b1b2_logits)
+        each_row = np.concatenate(tuple(each_row), axis=-1)
+        sim_matrix.append(each_row)
+    return sim_matrix
+
+def run(model, dataloader, args):
+    autocast = get_autocast(args.precision)
+    input_dtype = get_input_dtype(args.precision)
+
+    with torch.no_grad():
+        sequence_output_list, visual_output_list = [], []
+        for images, input_ids, attention_mask in tqdm(dataloader, unit_scale=args.batch_size):
+            images = images.to(device=args.device, dtype=input_dtype)
+            images = images.unsqueeze(2)
+            input_ids = input_ids.squeeze().to(args.device)
+            attention_mask = attention_mask.squeeze().to(args.device)
+
+            with autocast():
+                # predict
+                sequence_output = model.encode_text(input_ids, attention_mask)
+                visual_output = model.encode_image(images)
+            sequence_output_list.append(sequence_output)
+            visual_output_list.append(visual_output)
+    sim_matrix = _run_on_single_gpu(model, sequence_output_list, visual_output_list)
+    sim_matrix = np.concatenate(tuple(sim_matrix), axis=0)
+    return sim_matrix
+
+
+def zero_shot_eval(model, data, epoch, args):
+    temp_val_al_ret_data = args.val_al_ret_data
+    args.val_al_ret_data = list(data.keys())
+    assert len(args.val_al_ret_data) == 1
+    args.val_al_ret_data = args.val_al_ret_data[0]
+
+    if args.val_al_ret_data not in data:
+        return {}
+    if args.zeroshot_frequency == 0:
+        return {}
+    if (epoch % args.zeroshot_frequency) != 0 and epoch != args.epochs:
+        return {}
+    if args.distributed and not args.horovod:
+        model = model.module
+
+    logging.info(f'Starting zero-shot {args.val_al_ret_data.upper()}.')
+
+    results = {}
+    if args.val_al_ret_data in data:
+        logit_matrix = run(model, data[args.val_al_ret_data].dataloader, args)
+        results = compute_metrics(logit_matrix)
+
+    logging.info(f'Finished zero-shot {args.val_al_ret_data.upper()}.')
+
+    args.val_al_ret_data = temp_val_al_ret_data
+    return results

RAG/Knowledge_Database/languagebind_main/d_cls/cp_zero_shot_metadata.py

@@ -0,0 +1,117 @@
+import os
+
+import pandas as pd
+
+OPENAI_IMAGENET_TEMPLATES = (
+    lambda c: f'a bad photo of a {c}.',
+    lambda c: f'a photo of many {c}.',
+    lambda c: f'a sculpture of a {c}.',
+    lambda c: f'a photo of the hard to see {c}.',
+    lambda c: f'a low resolution photo of the {c}.',
+    lambda c: f'a rendering of a {c}.',
+    lambda c: f'graffiti of a {c}.',
+    lambda c: f'a bad photo of the {c}.',
+    lambda c: f'a cropped photo of the {c}.',
+    lambda c: f'a tattoo of a {c}.',
+    lambda c: f'the embroidered {c}.',
+    lambda c: f'a photo of a hard to see {c}.',
+    lambda c: f'a bright photo of a {c}.',
+    lambda c: f'a photo of a clean {c}.',
+    lambda c: f'a photo of a dirty {c}.',
+    lambda c: f'a dark photo of the {c}.',
+    lambda c: f'a drawing of a {c}.',
+    lambda c: f'a photo of my {c}.',
+    lambda c: f'the plastic {c}.',
+    lambda c: f'a photo of the cool {c}.',
+    lambda c: f'a close-up photo of a {c}.',
+    lambda c: f'a black and white photo of the {c}.',
+    lambda c: f'a painting of the {c}.',
+    lambda c: f'a painting of a {c}.',
+    lambda c: f'a pixelated photo of the {c}.',
+    lambda c: f'a sculpture of the {c}.',
+    lambda c: f'a bright photo of the {c}.',
+    lambda c: f'a cropped photo of a {c}.',
+    lambda c: f'a plastic {c}.',
+    lambda c: f'a photo of the dirty {c}.',
+    lambda c: f'a jpeg corrupted photo of a {c}.',
+    lambda c: f'a blurry photo of the {c}.',
+    lambda c: f'a photo of the {c}.',
+    lambda c: f'a good photo of the {c}.',
+    lambda c: f'a rendering of the {c}.',
+    lambda c: f'a {c} in a video game.',
+    lambda c: f'a photo of one {c}.',
+    lambda c: f'a doodle of a {c}.',
+    lambda c: f'a close-up photo of the {c}.',
+    lambda c: f'a photo of a {c}.',
+    lambda c: f'the origami {c}.',
+    lambda c: f'the {c} in a video game.',
+    lambda c: f'a sketch of a {c}.',
+    lambda c: f'a doodle of the {c}.',
+    lambda c: f'a origami {c}.',
+    lambda c: f'a low resolution photo of a {c}.',
+    lambda c: f'the toy {c}.',
+    lambda c: f'a rendition of the {c}.',
+    lambda c: f'a photo of the clean {c}.',
+    lambda c: f'a photo of a large {c}.',
+    lambda c: f'a rendition of a {c}.',
+    lambda c: f'a photo of a nice {c}.',
+    lambda c: f'a photo of a weird {c}.',
+    lambda c: f'a blurry photo of a {c}.',
+    lambda c: f'a cartoon {c}.',
+    lambda c: f'art of a {c}.',
+    lambda c: f'a sketch of the {c}.',
+    lambda c: f'a embroidered {c}.',
+    lambda c: f'a pixelated photo of a {c}.',
+    lambda c: f'itap of the {c}.',
+    lambda c: f'a jpeg corrupted photo of the {c}.',
+    lambda c: f'a good photo of a {c}.',
+    lambda c: f'a plushie {c}.',
+    lambda c: f'a photo of the nice {c}.',
+    lambda c: f'a photo of the small {c}.',
+    lambda c: f'a photo of the weird {c}.',
+    lambda c: f'the cartoon {c}.',
+    lambda c: f'art of the {c}.',
+    lambda c: f'a drawing of the {c}.',
+    lambda c: f'a photo of the large {c}.',
+    lambda c: f'a black and white photo of a {c}.',
+    lambda c: f'the plushie {c}.',
+    lambda c: f'a dark photo of a {c}.',
+    lambda c: f'itap of a {c}.',
+    lambda c: f'graffiti of the {c}.',
+    lambda c: f'a toy {c}.',
+    lambda c: f'itap of my {c}.',
+    lambda c: f'a photo of a cool {c}.',
+    lambda c: f'a photo of a small {c}.',
+    lambda c: f'a tattoo of the {c}.',
+)
+
+
+# a much smaller subset of above prompts
+# from https://github.com/openai/CLIP/blob/main/notebooks/Prompt_Engineering_for_ImageNet.ipynb
+SIMPLE_IMAGENET_TEMPLATES = (
+    lambda c: f'itap of a {c}.',
+    lambda c: f'a bad photo of the {c}.',
+    lambda c: f'a origami {c}.',
+    lambda c: f'a photo of the large {c}.',
+    lambda c: f'a {c} in a video game.',
+    lambda c: f'art of the {c}.',
+    lambda c: f'a photo of the small {c}.',
+)
+
+
+IMAGENET_CLASSNAMES = (
+
+)
+
+
+CLASSNAMES = {
+    'NYUV2': (
+        "bathroom", "bedroom", "bookstore", "classroom", "dining room",
+        "home office", "kitchen", "living room", "office", "others"
+    ),
+    'SUNRGBD': (
+        "bathroom", "bedroom", "classroom", "computer room", "conference room", "corridor", "dining area",
+        "dining room", "discussion area", "furniture store", "home office", "kitchen", "lab", "lecture theatre",
+        "library", "living room", "office", "rest space", "study space"
+    ),
+}

RAG/Knowledge_Database/languagebind_main/d_cls/datasets.py

@@ -0,0 +1,20 @@
+import cv2
+import torch
+
+from data.build_datasets import DataInfo
+from data.process_depth import get_depth_transform, opencv_loader
+from torchvision import datasets
+
+def get_depth_dataset(args):
+    data_path = args.depth_data_path
+    transform = get_depth_transform(args)
+    dataset = datasets.ImageFolder(data_path, transform=transform, loader=opencv_loader)
+
+    dataloader = torch.utils.data.DataLoader(
+        dataset,
+        batch_size=args.batch_size,
+        num_workers=args.workers,
+        sampler=None,
+    )
+
+    return DataInfo(dataloader=dataloader, sampler=None)

RAG/Knowledge_Database/languagebind_main/d_cls/precision.py

@@ -0,0 +1,12 @@
+import torch
+from contextlib import suppress
+
+
+def get_autocast(precision):
+    if precision == 'amp':
+        return torch.cuda.amp.autocast
+    elif precision == 'amp_bfloat16' or precision == 'amp_bf16':
+        # amp_bfloat16 is more stable than amp float16 for clip training
+        return lambda: torch.cuda.amp.autocast(dtype=torch.bfloat16)
+    else:
+        return suppress

RAG/Knowledge_Database/languagebind_main/d_cls/zero_shot.py

@@ -0,0 +1,90 @@
+import logging
+
+import torch
+import torch.nn.functional as F
+from tqdm import tqdm
+
+from open_clip import get_input_dtype, get_tokenizer
+from open_clip.factory import HF_HUB_PREFIX
+from .precision import get_autocast
+from .zero_shot_classifier import build_zero_shot_classifier
+from .zero_shot_metadata import CLASSNAMES, OPENAI_IMAGENET_TEMPLATES
+
+
+def accuracy(output, target, topk=(1,)):
+    pred = output.topk(max(topk), 1, True, True)[1].t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+    return [float(correct[:k].reshape(-1).float().sum(0, keepdim=True).cpu().numpy()) for k in topk]
+
+
+def run(model, classifier, dataloader, args):
+    autocast = get_autocast(args.precision)
+    input_dtype = get_input_dtype(args.precision)
+
+    with torch.no_grad():
+        top1, top5, n = 0., 0., 0.
+        for images, target in tqdm(dataloader, unit_scale=args.batch_size):
+            images = images.to(device=args.device, dtype=input_dtype)
+            images = images.unsqueeze(2)
+            target = target.to(args.device)
+
+            with autocast():
+                # predict
+                output = model(image=images)
+                image_features = output['image_features'] if isinstance(output, dict) else output[0]
+                logits = 100. * image_features @ classifier
+
+            # measure accuracy
+            acc1, acc5 = accuracy(logits, target, topk=(1, 5))
+            top1 += acc1
+            top5 += acc5
+            n += images.size(0)
+
+    top1 = (top1 / n)
+    top5 = (top5 / n)
+    return top1, top5
+
+
+def zero_shot_eval(model, data, epoch, args):
+    temp_val_d_cls_data = args.val_d_cls_data
+    args.val_d_cls_data = list(data.keys())
+    assert len(args.val_d_cls_data) == 1
+    args.val_d_cls_data = args.val_d_cls_data[0]
+
+    if args.val_d_cls_data not in data:
+        return {}
+    if args.zeroshot_frequency == 0:
+        return {}
+    if (epoch % args.zeroshot_frequency) != 0 and epoch != args.epochs:
+        return {}
+    if args.distributed and not args.horovod:
+        model = model.module
+
+    logging.info(f'Starting zero-shot {args.val_d_cls_data.upper()}.')
+
+    logging.info('Building zero-shot classifier')
+    autocast = get_autocast(args.precision)
+    with autocast():
+        tokenizer = get_tokenizer(HF_HUB_PREFIX+args.model, cache_dir=args.cache_dir)
+        # tokenizer = get_tokenizer("ViT-L-14")
+        classifier = build_zero_shot_classifier(
+            model,
+            tokenizer=tokenizer,
+            classnames=CLASSNAMES[args.val_d_cls_data],
+            templates=OPENAI_IMAGENET_TEMPLATES,
+            num_classes_per_batch=10,
+            device=args.device,
+            use_tqdm=True,
+        )
+
+    logging.info('Using classifier')
+    results = {}
+    if args.val_d_cls_data in data:
+        top1, top5 = run(model, classifier, data[args.val_d_cls_data].dataloader, args)
+        results[f'{args.val_d_cls_data}-zeroshot-val-top1'] = top1
+        results[f'{args.val_d_cls_data}-zeroshot-val-top5'] = top5
+
+    logging.info(f'Finished zero-shot {args.val_d_cls_data.upper()}.')
+
+    args.val_d_cls_data = temp_val_d_cls_data
+    return results

RAG/Knowledge_Database/languagebind_main/d_cls/zero_shot_classifier.py

@@ -0,0 +1,111 @@
+from functools import partial
+from itertools import islice
+from typing import Callable, List, Optional, Sequence, Union
+
+import torch
+import torch.nn.functional as F
+
+
+def batched(iterable, n):
+    """Batch data into lists of length *n*. The last batch may be shorter.
+    NOTE based on more-itertools impl, to be replaced by python 3.12 itertools.batched impl
+    """
+    it = iter(iterable)
+    while True:
+        batch = list(islice(it, n))
+        if not batch:
+            break
+        yield batch
+
+
+def build_zero_shot_classifier(
+        model,
+        tokenizer,
+        classnames: Sequence[str],
+        templates: Sequence[Union[Callable, str]],
+        num_classes_per_batch: Optional[int] = 10,
+        device: Union[str, torch.device] = 'cpu',
+        use_tqdm: bool = False,
+):
+    """ Build zero-shot classifier weights by iterating over class names in batches
+    Args:
+        model: CLIP model instance
+        tokenizer: CLIP tokenizer instance
+        classnames: A sequence of class (label) names
+        templates: A sequence of callables or format() friendly strings to produce templates per class name
+        num_classes_per_batch: The number of classes to batch together in each forward, all if None
+        device: Device to use.
+        use_tqdm: Enable TQDM progress bar.
+    """
+    assert isinstance(templates, Sequence) and len(templates) > 0
+    assert isinstance(classnames, Sequence) and len(classnames) > 0
+    use_format = isinstance(templates[0], str)
+    num_templates = len(templates)
+    num_classes = len(classnames)
+    if use_tqdm:
+        import tqdm
+        num_iter = 1 if num_classes_per_batch is None else ((num_classes - 1) // num_classes_per_batch + 1)
+        iter_wrap = partial(tqdm.tqdm, total=num_iter, unit_scale=num_classes_per_batch)
+    else:
+        iter_wrap = iter
+
+    def _process_batch(batch_classnames):
+        num_batch_classes = len(batch_classnames)
+        texts = [template.format(c) if use_format else template(c) for c in batch_classnames for template in templates]
+        input_ids, attention_mask = tokenizer(texts)
+        input_ids, attention_mask = input_ids.to(device), attention_mask.to(device)
+        class_embeddings = F.normalize(model.encode_text(input_ids, attention_mask), dim=-1)
+        class_embeddings = class_embeddings.reshape(num_batch_classes, num_templates, -1).mean(dim=1)
+        class_embeddings = class_embeddings / class_embeddings.norm(dim=1, keepdim=True)
+        class_embeddings = class_embeddings.T
+        return class_embeddings
+
+    with torch.no_grad():
+        if num_classes_per_batch:
+            batched_embeds = [_process_batch(batch) for batch in iter_wrap(batched(classnames, num_classes_per_batch))]
+            zeroshot_weights = torch.cat(batched_embeds, dim=1)
+        else:
+            zeroshot_weights = _process_batch(classnames)
+    return zeroshot_weights
+
+
+def build_zero_shot_classifier_legacy(
+        model,
+        tokenizer,
+        classnames: Sequence[str],
+        templates: Sequence[Union[Callable, str]],
+        device: Union[str, torch.device] = 'cpu',
+        use_tqdm: bool = False,
+):
+    """ Build zero-shot classifier weights by iterating over class names 1 by 1
+    Args:
+        model: CLIP model instance
+        tokenizer: CLIP tokenizer instance
+        classnames: A sequence of class (label) names
+        templates: A sequence of callables or format() friendly strings to produce templates per class name
+        device: Device to use.
+        use_tqdm: Enable TQDM progress bar.
+    """
+    assert isinstance(templates, Sequence) and len(templates) > 0
+    assert isinstance(classnames, Sequence) and len(classnames) > 0
+    if use_tqdm:
+        import tqdm
+        iter_wrap = tqdm.tqdm
+    else:
+        iter_wrap = iter
+
+    use_format = isinstance(templates[0], str)
+
+    with torch.no_grad():
+        zeroshot_weights = []
+        for classname in iter_wrap(classnames):
+            texts = [template.format(classname) if use_format else template(classname) for template in templates]
+            texts = tokenizer(texts).to(device)  # tokenize
+            class_embeddings = model.encode_text(texts)
+            class_embedding = F.normalize(class_embeddings, dim=-1).mean(dim=0)
+            class_embedding /= class_embedding.norm()
+            zeroshot_weights.append(class_embedding)
+        zeroshot_weights = torch.stack(zeroshot_weights, dim=1).to(device)
+
+    return zeroshot_weights
+

RAG/Knowledge_Database/languagebind_main/d_cls/zero_shot_metadata.py

@@ -0,0 +1,117 @@
+import os
+
+import pandas as pd
+
+OPENAI_IMAGENET_TEMPLATES = (
+    lambda c: f'a bad depth photo of a {c}.',
+    lambda c: f'a depth photo of many {c}.',
+    lambda c: f'a sculpture of a {c}.',
+    lambda c: f'a depth photo of the hard to see {c}.',
+    lambda c: f'a low resolution depth photo of the {c}.',
+    lambda c: f'a rendering of a {c}.',
+    lambda c: f'graffiti of a {c}.',
+    lambda c: f'a bad depth photo of the {c}.',
+    lambda c: f'a cropped depth photo of the {c}.',
+    lambda c: f'a tattoo of a {c}.',
+    lambda c: f'the embroidered {c}.',
+    lambda c: f'a depth photo of a hard to see {c}.',
+    lambda c: f'a bright depth photo of a {c}.',
+    lambda c: f'a depth photo of a clean {c}.',
+    lambda c: f'a depth photo of a dirty {c}.',
+    lambda c: f'a dark depth photo of the {c}.',
+    lambda c: f'a drawing of a {c}.',
+    lambda c: f'a depth photo of my {c}.',
+    lambda c: f'the plastic {c}.',
+    lambda c: f'a depth photo of the cool {c}.',
+    lambda c: f'a close-up depth photo of a {c}.',
+    lambda c: f'a black and white depth photo of the {c}.',
+    lambda c: f'a painting of the {c}.',
+    lambda c: f'a painting of a {c}.',
+    lambda c: f'a pixelated depth photo of the {c}.',
+    lambda c: f'a sculpture of the {c}.',
+    lambda c: f'a bright depth photo of the {c}.',
+    lambda c: f'a cropped depth photo of a {c}.',
+    lambda c: f'a plastic {c}.',
+    lambda c: f'a depth photo of the dirty {c}.',
+    lambda c: f'a jpeg corrupted depth photo of a {c}.',
+    lambda c: f'a blurry depth photo of the {c}.',
+    lambda c: f'a depth photo of the {c}.',
+    lambda c: f'a good depth photo of the {c}.',
+    lambda c: f'a rendering of the {c}.',
+    lambda c: f'a {c} in a video game.',
+    lambda c: f'a depth photo of one {c}.',
+    lambda c: f'a doodle of a {c}.',
+    lambda c: f'a close-up depth photo of the {c}.',
+    lambda c: f'a depth photo of a {c}.',
+    lambda c: f'the origami {c}.',
+    lambda c: f'the {c} in a video game.',
+    lambda c: f'a sketch of a {c}.',
+    lambda c: f'a doodle of the {c}.',
+    lambda c: f'a origami {c}.',
+    lambda c: f'a low resolution depth photo of a {c}.',
+    lambda c: f'the toy {c}.',
+    lambda c: f'a rendition of the {c}.',
+    lambda c: f'a depth photo of the clean {c}.',
+    lambda c: f'a depth photo of a large {c}.',
+    lambda c: f'a rendition of a {c}.',
+    lambda c: f'a depth photo of a nice {c}.',
+    lambda c: f'a depth photo of a weird {c}.',
+    lambda c: f'a blurry depth photo of a {c}.',
+    lambda c: f'a cartoon {c}.',
+    lambda c: f'art of a {c}.',
+    lambda c: f'a sketch of the {c}.',
+    lambda c: f'a embroidered {c}.',
+    lambda c: f'a pixelated depth photo of a {c}.',
+    lambda c: f'itap of the {c}.',
+    lambda c: f'a jpeg corrupted depth photo of the {c}.',
+    lambda c: f'a good depth photo of a {c}.',
+    lambda c: f'a plushie {c}.',
+    lambda c: f'a depth photo of the nice {c}.',
+    lambda c: f'a depth photo of the small {c}.',
+    lambda c: f'a depth photo of the weird {c}.',
+    lambda c: f'the cartoon {c}.',
+    lambda c: f'art of the {c}.',
+    lambda c: f'a drawing of the {c}.',
+    lambda c: f'a depth photo of the large {c}.',
+    lambda c: f'a black and white depth photo of a {c}.',
+    lambda c: f'the plushie {c}.',
+    lambda c: f'a dark depth photo of a {c}.',
+    lambda c: f'itap of a {c}.',
+    lambda c: f'graffiti of the {c}.',
+    lambda c: f'a toy {c}.',
+    lambda c: f'itap of my {c}.',
+    lambda c: f'a depth photo of a cool {c}.',
+    lambda c: f'a depth photo of a small {c}.',
+    lambda c: f'a tattoo of the {c}.',
+)
+
+
+# a much smaller subset of above prompts
+# from https://github.com/openai/CLIP/blob/main/notebooks/Prompt_Engineering_for_ImageNet.ipynb
+SIMPLE_IMAGENET_TEMPLATES = (
+    lambda c: f'itap of a {c}.',
+    lambda c: f'a bad depth photo of the {c}.',
+    lambda c: f'a origami {c}.',
+    lambda c: f'a depth photo of the large {c}.',
+    lambda c: f'a {c} in a video game.',
+    lambda c: f'art of the {c}.',
+    lambda c: f'a depth photo of the small {c}.',
+)
+
+
+IMAGENET_CLASSNAMES = (
+
+)
+
+
+CLASSNAMES = {
+    'NYUV2': (
+        "bathroom", "bedroom", "bookstore", "classroom", "dining room",
+        "home office", "kitchen", "living room", "office", "others"
+    ),
+    'SUNRGBD': (
+        "bathroom", "bedroom", "classroom", "computer room", "conference room", "corridor", "dining area",
+        "dining room", "discussion area", "furniture store", "home office", "kitchen", "lab", "lecture theatre",
+        "library", "living room", "office", "rest space", "study space"
+    ),
+}

RAG/Knowledge_Database/languagebind_main/d_cls/zeroshot_cls.py

@@ -0,0 +1,47 @@
+
+import json
+import logging
+import os
+from training.distributed import is_master
+from .zero_shot import zero_shot_eval
+
+try:
+    import wandb
+except ImportError:
+    wandb = None
+
+
+
+def evaluate_d_cls(model, data, epoch, args, tb_writer=None):
+    metrics = {}
+    if not is_master(args):
+        return metrics
+    model.eval()
+
+    zero_shot_metrics = zero_shot_eval(model, data, epoch, args)
+    metrics.update(zero_shot_metrics)
+
+    if not metrics:
+        return metrics
+
+    logging.info(
+        f"Eval Epoch: {epoch} "
+        + "\t".join([f"{k}: {round(v, 4):.4f}" for k, v in metrics.items()])
+    )
+
+    if args.save_logs:
+        for name, val in metrics.items():
+            if tb_writer is not None:
+                tb_writer.add_scalar(f"val/d_cls/{args.val_d_cls_data[0].lower()}/{name}", val, epoch)
+        args.d_cls_output_dir = os.path.join(args.log_base_path, f'd_cls/{args.val_d_cls_data[0].lower()}')
+        os.makedirs(args.d_cls_output_dir, exist_ok=True)
+        with open(os.path.join(args.d_cls_output_dir, "results.jsonl"), "a+") as f:
+            f.write(json.dumps(metrics))
+            f.write("\n")
+
+    if args.wandb:
+        assert wandb is not None, 'Please install wandb.'
+        for name, val in metrics.items():
+            wandb.log({f"val/{name}": val, 'epoch': epoch})
+
+    return metrics

RAG/Knowledge_Database/languagebind_main/data/base_datasets.py

@@ -0,0 +1,215 @@
+import contextlib
+import io
+import json
+import logging
+import os.path
+import random
+import re
+import time
+
+import pandas as pd
+
+from a_cls.dataloader import make_midname_dict
+from open_clip import get_tokenizer
+from open_clip.factory import HF_HUB_PREFIX
+from .process_video import load_and_transform_video, get_video_transform
+from .process_audio import load_and_transform_audio, get_audio_transform
+from .process_text import load_and_transform_text
+from .process_depth import load_and_transform_depth, get_depth_transform
+from .process_thermal import load_and_transform_thermal, get_thermal_transform
+
+import argparse
+from os.path import join as opj
+from torch.utils.data import Dataset, DataLoader
+from tqdm import tqdm
+
+
+
+class VAT_dataset(Dataset):
+    def __init__(self, args):
+        super().__init__()
+        self.video_decode_backend = args.video_decode_backend
+        self.num_frames = args.num_frames
+        self.text_type = args.text_type
+        self.total_text = ['raw', 'mplug', 'polish_mplug', 'sound_mplug'] + [f'ofa{i}' for i in range(8)]
+        self.weight = [0.2, 0.2, 0.2, 0.2] + [0.2 / 8] * 8
+        self.title = self.text_type == 'raw'
+        self.data_root = '/apdcephfs_cq3/share_1311970/A_Youtube'
+        if args.clip_type != 'al':
+            with open(args.train_data, 'r') as f:
+                self.id2title_folder_caps = json.load(f)
+            self.ids = list(self.id2title_folder_caps.keys())[:args.train_num_samples]
+        else:
+            self.id2path_cap, self.ids = get_audio_anno()
+
+        self.clip_type = args.clip_type
+
+        self.num_mel_bins = args.num_mel_bins
+        self.target_length = args.target_length
+        self.audio_sample_rate = args.audio_sample_rate
+        self.audio_mean = args.audio_mean
+        self.audio_std = args.audio_std
+
+        # self.audio_error_file = open('./audio_error_id.txt', 'w')
+
+        self.tokenizer = get_tokenizer(HF_HUB_PREFIX + args.model, cache_dir=args.cache_dir)
+        self.video_transform = get_video_transform(args)
+        self.audio_transform = get_audio_transform(args)
+        self.depth_transform = get_depth_transform(args)
+        self.thermal_transform = get_thermal_transform(args)
+
+    def __len__(self):
+        return len(self.ids)
+        # return self.id2title_folder_caps.shape[0]
+
+
+    def __getitem__(self, idx):
+        try:
+            if self.clip_type == 'al':
+                matched_modality, input_ids, attention_mask = self.get_audio_text(idx)
+                return matched_modality, input_ids, attention_mask
+            else:
+                id = self.ids[idx]
+                folder = self.id2title_folder_caps[id]['folder']
+                text_output, ofa_number = self.get_text(id)
+                input_ids, attention_mask = text_output['input_ids'], text_output['attention_mask']
+                if self.clip_type == 'vl' or self.clip_type == 'vl_new':
+                    matched_modality = self.get_video(id, folder)
+                # elif self.clip_type == 'al':
+                #     matched_modality = self.get_audio(id, folder)
+                elif self.clip_type == 'dl':
+                    matched_modality = self.get_depth(id, folder, ofa_number)
+                elif self.clip_type == 'tl':
+                    matched_modality = self.get_thermal(id, folder, ofa_number)
+                return matched_modality['pixel_values'], input_ids, attention_mask
+        except Exception as error_msg:
+            logging.info(f"Failed at {idx} with \"{error_msg}\"")
+            return self.__getitem__(random.randint(0, self.__len__()-1))
+
+    def get_video(self, id, folder):
+        # video_path = opj(self.data_root, folder, f'{id}.mp4')
+        resize_folder = 'new_download_resize256_skip15' if folder.startswith('new_') else f'{folder}_resize256_skip15'
+        video_path = opj(self.data_root, resize_folder, f'{id}.mp4')
+        video = load_and_transform_video(video_path, self.video_transform,
+                                         video_decode_backend=self.video_decode_backend, num_frames=self.num_frames)
+        return video
+
+    def get_audio_text(self, idx):
+
+        path_cap = self.id2path_cap[self.ids[idx]]
+        audio_path = path_cap['path']
+        audio_data = load_and_transform_audio(audio_path, self.audio_transform)
+
+        caption = path_cap['caption']
+        if isinstance(caption, list):
+            if isinstance(caption[0], str) and len(caption) > 1:
+                caption = random.choice(caption)
+            else:
+                caption = caption[0]
+
+        input_ids, attention_mask = self.tokenizer(caption)
+
+        return audio_data, input_ids.squeeze(), attention_mask.squeeze()
+
+        # def get_audio(self, idx):
+        '''
+        audio_path = opj(self.data_root, folder, f'{id}.mp3')
+        if os.path.exists(audio_path):
+            pass
+        else:
+            audio_path = audio_path[:-4] + '.m4a'
+            if os.path.exists(audio_path):
+                pass
+            else:
+                audio_path = audio_path[:-4] + '.wav'
+                if not os.path.exists(audio_path):
+                    # self.audio_error_file.write(audio_path[:-4] + '\n')
+                    raise FileNotFoundError(f'Not found audio file at \'{audio_path[:-4]}\' with .mp3 .m4a .wav')
+            # AudioSegment.from_file(audio_path).export(audio_path[:-4] + '.mp3', format='mp3')
+            # audio_path = opj(self.data_root, folder, f'{id}.mp3')
+        audio = load_and_transform_audio(audio_path, self.audio_transform)
+        '''
+
+        # audio_path = opj(self.data_root, folder+'_ffmpeg_mp3', f'{id}.mp3')
+        # audio = load_and_transform_audio(audio_path, self.audio_transform)
+
+
+        '''
+        audiocap_id = self.meta['uniq_id'][idx]
+        audio_path = f'/apdcephfs_cq3/share_1311970/downstream_datasets/Audio/audiocaps/audio/train/{audiocap_id}.flac'
+        audio_data = load_and_transform_audio(audio_path, self.audio_transform)
+
+        caption = self.meta['text'][idx]
+        input_ids, attention_mask = self.tokenizer(caption)
+        return audio_data, input_ids.squeeze(), attention_mask.squeeze()
+        '''
+
+        '''
+        path_cap = self.id2path_cap[self.ids[idx]]
+        audio_path = f"/remote-home/freesound/{path_cap['path']}"
+        audio_data = load_and_transform_audio(audio_path, self.audio_transform)
+
+        caption = path_cap['caption']
+        input_ids, attention_mask = self.tokenizer(caption)
+        '''
+
+        # return audio
+
+
+    def get_text(self, id):
+        if self.text_type != 'mix':
+            text = self.id2title_folder_caps[id][self.text_type]
+            text_output = load_and_transform_text(text, self.tokenizer, title=self.title)
+            return text_output, None
+        else:
+            text_type = random.choices(self.total_text, self.weight)[0]
+            ofa_number = None
+            if text_type.startswith('ofa'):
+                ofa_number = int(text_type[-1])
+                text = self.id2title_folder_caps[id]['ofa'][ofa_number]
+            else:
+                text = self.id2title_folder_caps[id][text_type]
+            text_output = load_and_transform_text(text, self.tokenizer, title=text_type=='raw')
+            return text_output, ofa_number
+
+    def get_depth(self, id, folder, ofa_number):
+        depth_folder = opj(self.data_root, folder, f'{id}_depth_f8glpn_folder')
+        random_id = random.randint(0, 7) if ofa_number is None else ofa_number
+        # random_id = 3
+        depth_path = os.path.join(depth_folder, f'{random_id}.png')
+        depth = load_and_transform_depth(depth_path, self.depth_transform)
+        return depth
+
+    def get_thermal(self, id, folder, ofa_number):
+        thermal_folder = opj(self.data_root, folder, f'{id}_thermal_folder')
+        random_id = random.randint(0, 7) if ofa_number is None else ofa_number
+        # random_id = 3
+        thermal_path = os.path.join(thermal_folder, f'{random_id}.jpg')
+        thermal = load_and_transform_thermal(thermal_path, self.thermal_transform)
+        return thermal
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser('Pre-training', add_help=False)
+    parser.add_argument('--num_frames', default=8, type=float, help='')
+    parser.add_argument('--workers', default=10, type=int, help='')
+    args = parser.parse_args()
+
+    args.cache_dir = 'D:\Omni-modal-hf'
+    args.num_frames = 8
+    args.clip_type = 'vl'
+    args.num_mel_bins = 128
+    args.target_length = 1024
+    args.audio_sample_rate = 16000
+    args.audio_mean = 1
+    args.audio_std = 1
+    args.rank = 0
+    args.batch_size = 16
+
+    train_dataset = VAT_dataset(args)
+    load = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.workers)
+
+    for samples in tqdm((load)):
+        matched_modality, input_ids, attention_mask = samples
+        # print(video.shape, text.shape)

RAG/Knowledge_Database/languagebind_main/data/bpe_simple_vocab_16e6.txt.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a
+size 1356917

RAG/Knowledge_Database/languagebind_main/data/build_datasets.py

@@ -0,0 +1,247 @@
+import os
+import time
+from dataclasses import dataclass
+from multiprocessing import Value
+
+import torch
+from torch.utils.data import DataLoader
+from torch.utils.data.distributed import DistributedSampler
+
+from data.base_datasets import VAT_dataset
+from data.new_loadvat import get_wds_dataset
+from open_clip import get_tokenizer
+from open_clip.factory import HF_HUB_PREFIX
+
+
+class SharedEpoch:
+    def __init__(self, epoch: int = 0):
+        self.shared_epoch = Value('i', epoch)
+
+    def set_value(self, epoch):
+        self.shared_epoch.value = epoch
+
+    def get_value(self):
+        return self.shared_epoch.value
+
+@dataclass
+class DataInfo:
+    dataloader: DataLoader
+    sampler: DistributedSampler = None
+    shared_epoch: SharedEpoch = None
+
+    def set_epoch(self, epoch):
+        if self.shared_epoch is not None:
+            self.shared_epoch.set_value(epoch)
+        if self.sampler is not None and isinstance(self.sampler, DistributedSampler):
+            self.sampler.set_epoch(epoch)
+
+def get_VAT_dataset(args):
+    dataset = VAT_dataset(args)
+    num_samples = len(dataset)
+    sampler = DistributedSampler(dataset) if args.distributed else None
+    shuffle = sampler is None
+
+    dataloader = DataLoader(
+        dataset,
+        batch_size=args.batch_size,
+        # prefetch_factor=2,
+        # persistent_workers=True,
+        shuffle=shuffle,
+        num_workers=args.workers,
+        pin_memory=True,
+        sampler=sampler,
+        drop_last=True,
+    )
+    dataloader.num_samples = num_samples
+    dataloader.num_batches = len(dataloader)
+
+    return DataInfo(dataloader, sampler)
+
+def get_data(args, epoch=0):
+    data = {}
+
+    if args.do_train:
+        if args.train_data.endswith(".json"):
+            data[f"{args.clip_type}_pt"] = get_VAT_dataset(args)
+        elif args.train_data.endswith(".tar"):
+            data[f"{args.clip_type}_pt"] = get_wds_dataset(args, is_train=True, epoch=epoch)
+        else:
+            raise NameError
+
+    if args.do_eval:
+        temp_batch_size = args.batch_size
+        args.batch_size = 8 if args.val_vl_ret_data else 16
+        data_root = "/apdcephfs_cq3/share_1311970/downstream_datasets/VideoTextRetrieval/vtRetdata"
+        if args.val_vl_ret_data:
+            data["vl_ret"] = []
+            for val_vl_ret_data in args.val_vl_ret_data:
+                if val_vl_ret_data == "msrvtt":
+                    args.train_csv = os.path.join(f'{data_root}/MSRVTT/MSRVTT_train.9k.csv')
+                    args.val_csv = os.path.join(f'{data_root}/MSRVTT/MSRVTT_JSFUSION_test.csv')
+                    args.data_path = os.path.join(f'{data_root}/MSRVTT/MSRVTT_data.json')
+                    args.features_path = os.path.join(f'{data_root}/MSRVTT/MSRVTT_Videos')
+                elif val_vl_ret_data == "msvd":
+                    args.data_path = os.path.join(f'{data_root}/MSVD')
+                    args.features_path = os.path.join(f'{data_root}/MSVD/MSVD_Videos')
+                elif val_vl_ret_data == "activity":
+                    args.data_path = os.path.join(f'{data_root}/ActivityNet')
+                    args.features_path = os.path.join(f'{data_root}/ActivityNet/Videos/Activity_Videos')
+                elif val_vl_ret_data == "didemo":
+                    args.data_path = os.path.join(f'{data_root}/Didemo')
+                    args.features_path = os.path.join(f'{data_root}/Didemo/videos')
+                else:
+                    raise NameError
+
+                args.batch_size_val = args.batch_size if args.batch_size_val == 0 else args.batch_size_val
+                args.max_frames = args.num_frames
+                args.num_thread_reader = args.workers
+                args.slice_framepos = 2   # "0: cut from head frames; 1: cut from tail frames; 2: extract frames uniformly."
+
+                from vl_ret.data_dataloaders import DATALOADER_DICT
+
+                tokenizer = get_tokenizer(HF_HUB_PREFIX + args.model, cache_dir=args.cache_dir)
+                test_dataloader, test_length = None, 0
+                if DATALOADER_DICT[val_vl_ret_data]["test"] is not None:
+                    test_dataloader, test_length = DATALOADER_DICT[val_vl_ret_data]["test"](args, tokenizer)
+
+                if DATALOADER_DICT[val_vl_ret_data]["val"] is not None:
+                    val_dataloader, val_length = DATALOADER_DICT[val_vl_ret_data]["val"](args, tokenizer, subset="val")
+                else:
+                    val_dataloader, val_length = test_dataloader, test_length
+                ## report validation results if the ["test"] is None
+                if test_dataloader is None:
+                    test_dataloader, test_length = val_dataloader, val_length
+
+                data["vl_ret"].append({val_vl_ret_data: test_dataloader})
+
+        if args.val_v_cls_data:
+            data["v_cls"] = []
+            temp_val_v_cls_data = args.val_v_cls_data
+            for val_v_cls_data in temp_val_v_cls_data:
+                from v_cls import get_video_cls_dataloader
+                args.val_v_cls_data = val_v_cls_data
+                if args.val_v_cls_data == 'Kinetics-400':
+                    args.video_data_path = "/apdcephfs_cq3/share_1311970/downstream_datasets/VideoCls/new_k400/Kinetics-400/raw/Kinetics-400"
+                    args.nb_classes = 400
+                elif args.val_v_cls_data == 'Kinetics-600':
+                    args.video_data_path = "/apdcephfs_cq3/share_1311970/downstream_datasets/VideoCls/new_k600/Kinetics600/raw/Kinetics600"
+                    args.nb_classes = 600
+                args.data_root = args.video_data_path
+                args.data_set = val_v_cls_data
+                args.dist_eval = True
+                args.sampling_rate = 8
+                args.num_sample = 1
+                args.test_num_segment = 5
+                args.test_num_crop = 3
+                args.num_workers = args.workers
+                data['v_cls'].append({val_v_cls_data: get_video_cls_dataloader(args)})
+            args.val_v_cls_data = temp_val_v_cls_data
+
+        if args.val_a_cls_data:
+            temp_audio_mean, temp_audio_std = args.audio_mean, args.audio_std
+            args.audio_mean, args.audio_std = -4.2677393, 4.5689974
+            data["a_cls"] = []
+            data_root = "/apdcephfs_cq3/share_1311970/downstream_datasets/Audio"
+            temp_val_a_cls_data = args.val_a_cls_data
+            for val_a_cls_data in temp_val_a_cls_data:
+                from a_cls.datasets import get_audio_dataset
+                args.val_a_cls_data = val_a_cls_data
+                args.audio_data_path = os.path.join(data_root, f'{val_a_cls_data.lower()}/test')
+                data['a_cls'].append({val_a_cls_data: get_audio_dataset(args)})
+            args.val_a_cls_data = temp_val_a_cls_data
+            args.audio_mean, args.audio_mean = temp_audio_mean, temp_audio_std
+
+        if args.val_al_ret_data:
+            temp_audio_mean, temp_audio_std = args.audio_mean, args.audio_std
+            args.audio_mean, args.audio_std = -4.2677393, 4.5689974
+
+            data["al_ret"] = []
+            data_root = "/apdcephfs_cq3/share_1311970/downstream_datasets/Audio"
+            temp_val_al_ret_data = args.val_al_ret_data
+            for val_al_ret_data in temp_val_al_ret_data:
+                from al_ret.datasets import get_audio_dataset
+                args.val_al_ret_data = val_al_ret_data
+                if val_al_ret_data.lower() != 'msrvtt':
+                    args.audio_data_path = os.path.join(data_root, val_al_ret_data.lower())
+                    data['al_ret'].append({val_al_ret_data: get_audio_dataset(args)})
+                elif val_al_ret_data.lower() == 'msrvtt':
+                    args.train_csv = os.path.join(f'/apdcephfs_cq3/share_1311970/downstream_datasets/VideoTextRetrieval/vtRetdata/MSRVTT/MSRVTT_train.9k.csv')
+                    args.val_csv = os.path.join(f'/apdcephfs_cq3/share_1311970/downstream_datasets/VideoTextRetrieval/Audio/MSRVTT/MSRVTT_AUDIO_test.csv')
+                    args.data_path = os.path.join(f'/apdcephfs_cq3/share_1311970/downstream_datasets/VideoTextRetrieval/vtRetdata/MSRVTT/MSRVTT_data.json')
+                    args.features_path = os.path.join(f'/apdcephfs_cq3/share_1311970/downstream_datasets/VideoTextRetrieval/Audio/MSRVTT/videos/all')
+
+
+                    args.num_thread_reader = args.workers
+                    from al_ret.data_dataloaders import DATALOADER_DICT
+                    args.batch_size_val = args.batch_size if args.batch_size_val == 0 else args.batch_size_val
+
+                    tokenizer = get_tokenizer(HF_HUB_PREFIX + args.model, cache_dir=args.cache_dir)
+                    test_dataloader, test_length = None, 0
+                    if DATALOADER_DICT[val_al_ret_data.lower()]["test"] is not None:
+                        test_dataloader, test_length = DATALOADER_DICT[val_al_ret_data.lower()]["test"](args, tokenizer)
+
+                    if DATALOADER_DICT[val_al_ret_data.lower()]["val"] is not None:
+                        val_dataloader, val_length = DATALOADER_DICT[val_al_ret_data.lower()]["val"](args, tokenizer, subset="val")
+                    else:
+                        val_dataloader, val_length = test_dataloader, test_length
+                    ## report validation results if the ["test"] is None
+                    if test_dataloader is None:
+                        test_dataloader, test_length = val_dataloader, val_length
+                    data['al_ret'].append({val_al_ret_data: test_dataloader})
+
+            args.val_al_ret_data = temp_val_al_ret_data
+            args.audio_mean, args.audio_mean = temp_audio_mean, temp_audio_std
+
+        if args.val_a_cls_data:
+            temp_audio_mean, temp_audio_std = args.audio_mean, args.audio_std
+            args.audio_mean, args.audio_std = -4.2677393, 4.5689974
+            data["a_cls"] = []
+            data_root = "/apdcephfs_cq3/share_1311970/downstream_datasets/Audio"
+            temp_val_a_cls_data = args.val_a_cls_data
+            for val_a_cls_data in temp_val_a_cls_data:
+                from a_cls.datasets import get_audio_dataset
+                args.val_a_cls_data = val_a_cls_data
+                args.audio_data_path = os.path.join(data_root, f'{val_a_cls_data.lower()}/test')
+                data['a_cls'].append({val_a_cls_data: get_audio_dataset(args)})
+            args.val_a_cls_data = temp_val_a_cls_data
+            args.audio_mean, args.audio_mean = temp_audio_mean, temp_audio_std
+
+        if args.imagenet_val is not None:
+            from i_cls.datasets import get_imagenet
+            data['i_cls'] = {}
+            data['i_cls']["imagenet-val"] = get_imagenet(args, "val")
+        if args.imagenet_v2 is not None:
+            from i_cls.datasets import get_imagenet
+            if data.get('i_cls', None) is None:
+                data['i_cls'] = {}
+            data['i_cls']["imagenet-v2"] = get_imagenet(args, "v2")
+
+        if args.val_d_cls_data:
+            data["d_cls"] = []
+            data_root = "/apdcephfs_cq3/share_1311970/downstream_datasets/Depth"
+            temp_val_d_cls_data = args.val_d_cls_data
+            for val_d_cls_data in temp_val_d_cls_data:
+                from d_cls.datasets import get_depth_dataset
+                args.val_d_cls_data = val_d_cls_data
+                args.depth_data_path = os.path.join(data_root, f'{val_d_cls_data.lower()}/data/val')
+                data['d_cls'].append({val_d_cls_data: get_depth_dataset(args)})
+            args.val_d_cls_data = temp_val_d_cls_data
+
+
+        if args.val_t_cls_data:
+            data["t_cls"] = []
+            data_root = "/apdcephfs_cq3/share_1311970/downstream_datasets/Thermal"
+            temp_val_t_cls_data = args.val_t_cls_data
+            for val_t_cls_data in temp_val_t_cls_data:
+                from t_cls.datasets import get_thermal_dataset
+                args.val_t_cls_data = val_t_cls_data
+                args.thermal_data_path = os.path.join(data_root, f'{val_t_cls_data.lower()}/val')
+                data['t_cls'].append({val_t_cls_data: get_thermal_dataset(args)})
+            args.val_t_cls_data = temp_val_t_cls_data
+
+        args.batch_size = temp_batch_size
+
+    return data
+
+
+

RAG/Knowledge_Database/languagebind_main/data/new_loadvat.py

@@ -0,0 +1,498 @@
+import ast
+import io
+import json
+import logging
+import math
+import os
+import random
+import sys
+import braceexpand
+from dataclasses import dataclass
+from multiprocessing import Value
+
+import numpy.lib.format
+import numpy as np
+import pandas as pd
+import torch
+import torchvision.datasets as datasets
+import webdataset as wds
+from PIL import Image
+from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler, IterableDataset, get_worker_info
+from torch.utils.data.distributed import DistributedSampler
+from torchvision.transforms import ToTensor
+from tqdm import tqdm
+from webdataset.filters import _shuffle
+from webdataset.tariterators import base_plus_ext, url_opener, tar_file_expander, valid_sample
+
+from open_clip import get_tokenizer
+from open_clip.factory import HF_HUB_PREFIX
+from training.params import parse_args
+from data.process_text import load_and_transform_text
+from data.process_video import get_video_transform
+from data.process_audio import get_audio_transform
+from data.process_depth import get_depth_transform
+from data.process_thermal import get_thermal_transform
+import pdb
+try:
+    import horovod.torch as hvd
+except ImportError:
+    hvd = None
+
+
+
+class SharedEpoch:
+    def __init__(self, epoch: int = 0):
+        self.shared_epoch = Value('i', epoch)
+
+    def set_value(self, epoch):
+        self.shared_epoch.value = epoch
+
+    def get_value(self):
+        return self.shared_epoch.value
+
+
+@dataclass
+class DataInfo:
+    dataloader: DataLoader
+    sampler: DistributedSampler = None
+    shared_epoch: SharedEpoch = None
+
+    def set_epoch(self, epoch):
+        if self.shared_epoch is not None:
+            self.shared_epoch.set_value(epoch)
+        if self.sampler is not None and isinstance(self.sampler, DistributedSampler):
+            self.sampler.set_epoch(epoch)
+
+
+def expand_urls(urls, weights=None):
+    if weights is None:
+        expanded_urls = wds.shardlists.expand_urls(urls)
+        return expanded_urls, None
+    if isinstance(urls, str):
+        urllist = urls.split("::")
+        weights = weights.split('::')
+        assert len(weights) == len(urllist), \
+            f"Expected the number of data components ({len(urllist)}) and weights({len(weights)}) to match."
+        weights = [float(weight) for weight in weights]
+        all_urls, all_weights = [], []
+        for url, weight in zip(urllist, weights):
+            expanded_url = list(braceexpand.braceexpand(url))
+            expanded_weights = [weight for _ in expanded_url]
+            all_urls.extend(expanded_url)
+            all_weights.extend(expanded_weights)
+        return all_urls, all_weights
+    else:
+        all_urls = list(urls)
+        return all_urls, weights
+
+
+def get_dataset_size(shards):
+    shards_list, _ = expand_urls(shards)
+    dir_path = os.path.dirname(shards_list[0])
+    sizes_filename = os.path.join(dir_path, 'sizes.json')
+    len_filename = os.path.join(dir_path, '__len__')
+    if os.path.exists(sizes_filename):
+        sizes = json.load(open(sizes_filename, 'r'))
+        total_size = sum([int(sizes[os.path.basename(shard)]) for shard in shards_list])
+    elif os.path.exists(len_filename):
+        # FIXME this used to be eval(open(...)) but that seemed rather unsafe
+        total_size = ast.literal_eval(open(len_filename, 'r').read())
+    else:
+        total_size = None  # num samples undefined
+        # some common dataset sizes (at time of authors last download)
+        # CC3M (train): 2905954
+        # CC12M: 10968539
+        # LAION-400M: 407332084
+        # LAION-2B (english): 2170337258
+    num_shards = len(shards_list)
+    return total_size, num_shards
+
+
+
+def count_samples(dataloader):
+    os.environ["WDS_EPOCH"] = "0"
+    n_elements, n_batches = 0, 0
+    for images, texts in dataloader:
+        n_batches += 1
+        n_elements += len(images)
+        assert len(images) == len(texts)
+    return n_elements, n_batches
+
+
+def filter_no_caption_or_no_image(sample):
+    has_caption = ('raw.txt' in sample and 'mplug.txt' in sample and 'polish_mplug.txt' in sample and 'ofa3.txt' in sample)
+    has_image = ('frm7.jpg' in sample and 'tml0.jpg' in sample and 'dep0.npy' in sample)
+    return has_caption and has_image
+
+
+def log_and_continue(exn):
+    """Call in an exception handler to ignore any exception, issue a warning, and continue."""
+    logging.warning(f'Handling webdataset error ({repr(exn)}). Ignoring.')
+    return True
+
+
+def group_by_keys_nothrow(data, keys=base_plus_ext, lcase=True, suffixes=None, handler=None):
+    """Return function over iterator that groups key, value pairs into samples.
+
+    :param keys: function that splits the key into key and extension (base_plus_ext)
+    :param lcase: convert suffixes to lower case (Default value = True)
+    """
+    current_sample = None
+    for filesample in data:
+        assert isinstance(filesample, dict)
+        fname, value = filesample["fname"], filesample["data"]
+        prefix, suffix = keys(fname)
+        if prefix is None:
+            continue
+        if lcase:
+            suffix = suffix.lower()
+        # FIXME webdataset version throws if suffix in current_sample, but we have a potential for
+        #  this happening in the current LAION400m dataset if a tar ends with same prefix as the next
+        #  begins, rare, but can happen since prefix aren't unique across tar files in that dataset
+        if current_sample is None or prefix != current_sample["__key__"] or suffix in current_sample:
+            if valid_sample(current_sample):
+                yield current_sample
+            current_sample = dict(__key__=prefix, __url__=filesample["__url__"])
+        if suffixes is None or suffix in suffixes:
+            current_sample[suffix] = value
+    if valid_sample(current_sample):
+        yield current_sample
+
+
+def tarfile_to_samples_nothrow(src, handler=log_and_continue):
+    # NOTE this is a re-impl of the webdataset impl with group_by_keys that doesn't throw
+    streams = url_opener(src, handler=handler)
+    files = tar_file_expander(streams, handler=handler)
+    samples = group_by_keys_nothrow(files, handler=handler)
+    return samples
+
+
+def pytorch_worker_seed(increment=0):
+    """get dataloader worker seed from pytorch"""
+    worker_info = get_worker_info()
+    if worker_info is not None:
+        # favour using the seed already created for pytorch dataloader workers if it exists
+        seed = worker_info.seed
+        if increment:
+            # space out seed increments so they can't overlap across workers in different iterations
+            seed += increment * max(1, worker_info.num_workers)
+        return seed
+    # fallback to wds rank based seed
+    return wds.utils.pytorch_worker_seed()
+
+
+_SHARD_SHUFFLE_SIZE = 200
+_SHARD_SHUFFLE_INITIAL = 50
+_SAMPLE_SHUFFLE_SIZE = 500
+_SAMPLE_SHUFFLE_INITIAL = 100
+
+
+class detshuffle2(wds.PipelineStage):
+    def __init__(
+            self,
+            bufsize=1000,
+            initial=100,
+            seed=0,
+            epoch=-1,
+    ):
+        self.bufsize = bufsize
+        self.initial = initial
+        self.seed = seed
+        self.epoch = epoch
+
+    def run(self, src):
+        if isinstance(self.epoch, SharedEpoch):
+            epoch = self.epoch.get_value()
+        else:
+            # NOTE: this is epoch tracking is problematic in a multiprocess (dataloader workers or train)
+            # situation as different workers may wrap at different times (or not at all).
+            self.epoch += 1
+            epoch = self.epoch
+        rng = random.Random()
+        if self.seed < 0:
+            # If seed is negative, we use the worker's seed, this will be different across all nodes/workers
+            seed = pytorch_worker_seed(epoch)
+        else:
+            # This seed to be deterministic AND the same across all nodes/workers in each epoch
+            seed = self.seed + epoch
+        rng.seed(seed)
+        return _shuffle(src, self.bufsize, self.initial, rng)
+
+
+class ResampledShards2(IterableDataset):
+    """An iterable dataset yielding a list of urls."""
+
+    def __init__(
+            self,
+            urls,
+            weights=None,
+            nshards=sys.maxsize,
+            worker_seed=None,
+            deterministic=False,
+            epoch=-1,
+    ):
+        """Sample shards from the shard list with replacement.
+
+        :param urls: a list of URLs as a Python list or brace notation string
+        """
+        super().__init__()
+        urls, weights = expand_urls(urls, weights)
+        self.urls = urls
+        self.weights = weights
+        if self.weights is not None:
+            assert len(self.urls) == len(self.weights), \
+                f"Number of urls {len(self.urls)} and weights {len(self.weights)} should match."
+        assert isinstance(self.urls[0], str)
+        self.nshards = nshards
+        self.rng = random.Random()
+        self.worker_seed = worker_seed
+        self.deterministic = deterministic
+        self.epoch = epoch
+
+    def __iter__(self):
+        """Return an iterator over the shards."""
+        if isinstance(self.epoch, SharedEpoch):
+            epoch = self.epoch.get_value()
+        else:
+            # NOTE: this is epoch tracking is problematic in a multiprocess (dataloader workers or train)
+            # situation as different workers may wrap at different times (or not at all).
+            self.epoch += 1
+            epoch = self.epoch
+        if self.deterministic:
+            # reset seed w/ epoch if deterministic
+            if self.worker_seed is None:
+                # pytorch worker seed should be deterministic due to being init by arg.seed + rank + worker id
+                seed = pytorch_worker_seed(epoch)
+            else:
+                seed = self.worker_seed() + epoch
+            self.rng.seed(seed)
+        for _ in range(self.nshards):
+            if self.weights is None:
+                yield dict(url=self.rng.choice(self.urls))
+            else:
+                yield dict(url=self.rng.choices(self.urls, weights=self.weights, k=1)[0])
+
+
+class Decode:
+    def __init__(self, args=None):
+        self.num_frames = args.num_frames
+        self.text_type = args.text_type
+        self.chatgpt = self.text_type == 'polish_mplug'
+        self.title = self.text_type == 'raw'
+        self.clip_type = args.clip_type
+        self.tokenizer = get_tokenizer(HF_HUB_PREFIX + args.model, cache_dir=args.cache_dir)
+        self.video_transform = get_video_transform(args)
+        self.audio_transform = get_audio_transform(args)
+        self.depth_transform = get_depth_transform(args)
+        self.thermal_transform = get_thermal_transform(args)
+
+
+    def __call__(self, sample):
+        input_ids, attention_mask = self.get_text(sample[f"{self.text_type}.txt"], chatgpt=self.chatgpt, title=self.title)
+        if self.clip_type == 'vl':
+            matched_modality = self.get_video([sample[f"frm{i}.jpg"] for i in range(self.num_frames)])
+        elif self.clip_type == 'al':
+            matched_modality = self.get_audio()
+        elif self.clip_type == 'dl':
+            matched_modality = self.get_depth(sample[f"dep0.npy"])
+        elif self.clip_type == 'tl':
+            matched_modality = self.get_thermal(sample[f"tml0.jpg"])
+            # matched_modality = self.get_thermal(sample[f"tml{random.randint(0, 7)}.jpg"])
+        else:
+            raise ValueError
+        return matched_modality, input_ids, attention_mask
+
+
+    def get_video(self, frames):
+        video_data = []
+        for frame in frames:
+            with io.BytesIO(frame) as stream:
+                img = Image.open(stream)
+                img.load()
+            assert min(img.size) == 256
+            result = ToTensor()(img)
+            video_data.append(result)
+        video_data = torch.stack(video_data, dim=1)
+        # video_data torch.Size([3, 8, 455, 256])
+        # video_outputs torch.Size([3, 8, 224, 224])
+        video_outputs = self.video_transform(video_data)
+        return video_outputs
+
+
+    def get_text(self, text, chatgpt=True, title=False):
+        text = text.decode("utf-8")
+        if chatgpt:
+            assert text.startswith('In the video, ')
+            text = text[14:]
+        tokens = load_and_transform_text(text, self.tokenizer, title=title)
+        return tokens['input_ids'], tokens['attention_mask']
+
+    def get_audio(self):
+        raise NotImplementedError
+
+    def get_depth(self, depth):
+        stream = io.BytesIO(depth)
+        img = numpy.lib.format.read_array(stream)
+        depth = self.depth_transform(img)
+        return depth
+
+    def get_thermal(self, thermal):
+        with io.BytesIO(thermal) as stream:
+            img = Image.open(stream)
+            img.load()
+        thermal = self.thermal_transform(img)
+        return thermal
+
+def get_wds_dataset(args, is_train, epoch=0, floor=False):
+    input_shards = args.train_data if is_train else args.val_data
+    assert input_shards is not None
+    resampled = getattr(args, 'dataset_resampled', False) and is_train
+
+    num_shards = None
+    if is_train:
+        if args.train_num_samples is not None:
+            num_samples = args.train_num_samples
+        else:
+            num_samples, num_shards = get_dataset_size(input_shards)
+            if not num_samples:
+                raise RuntimeError(
+                    'Currently, the number of dataset samples must be specified for the training dataset. '
+                    'Please specify it via `--train-num-samples` if no dataset length info is present.')
+    else:
+        # Eval will just exhaust the iterator if the size is not specified.
+        num_samples = args.val_num_samples or 0
+
+    shared_epoch = SharedEpoch(epoch=epoch)  # create a shared epoch store to sync epoch to dataloader worker proc
+
+    if resampled:
+        pipeline = [ResampledShards2(
+            input_shards,
+            weights=args.train_data_upsampling_factors,
+            deterministic=True,
+            epoch=shared_epoch,
+        )]
+    else:
+        assert args.train_data_upsampling_factors is None, \
+            "--train_data_upsampling_factors is only supported when sampling with replacement (with --dataset-resampled)."
+        pipeline = [wds.SimpleShardList(input_shards)]
+
+    # at this point we have an iterator over all the shards
+    if is_train:
+        if not resampled:
+            pipeline.extend([
+                detshuffle2(
+                    bufsize=_SHARD_SHUFFLE_SIZE,
+                    initial=_SHARD_SHUFFLE_INITIAL,
+                    seed=args.seed,
+                    epoch=shared_epoch,
+                ),
+                wds.split_by_node,
+                wds.split_by_worker,
+            ])
+        pipeline.extend([
+            # at this point, we have an iterator over the shards assigned to each worker at each node
+            tarfile_to_samples_nothrow,  # wds.tarfile_to_samples(handler=log_and_continue),
+            wds.shuffle(
+                bufsize=_SAMPLE_SHUFFLE_SIZE,
+                initial=_SAMPLE_SHUFFLE_INITIAL,
+            ),
+        ])
+    else:
+        pipeline.extend([
+            wds.split_by_worker,
+            # at this point, we have an iterator over the shards assigned to each worker
+            wds.tarfile_to_samples(handler=log_and_continue),
+        ])
+    pipeline.extend([
+        wds.select(filter_no_caption_or_no_image),
+        # wds.decode("pilrgb", handler=log_and_continue),
+        # wds.rename(image="jpg;png;jpeg;webp", text="txt"),
+        # wds.map_dict(image=preprocess_img, text=lambda text: tokenizer(text)[0]),
+        # wds.to_tuple("image", "text"),
+        wds.map(Decode(args), handler=log_and_continue),
+        wds.batched(args.batch_size, partial=not is_train)
+    ])
+
+    dataset = wds.DataPipeline(*pipeline)
+
+    if is_train:
+        if not resampled:
+            num_shards = num_shards or len(expand_urls(input_shards)[0])
+            assert num_shards >= args.workers * args.world_size, 'number of shards must be >= total workers'
+        # roll over and repeat a few samples to get same number of full batches on each node
+        round_fn = math.floor if floor else math.ceil
+        global_batch_size = args.batch_size * args.world_size
+        num_batches = round_fn(num_samples / global_batch_size)
+        num_workers = max(1, args.workers)
+        num_worker_batches = round_fn(num_batches / num_workers)  # per dataloader worker
+        num_batches = num_worker_batches * num_workers
+        num_samples = num_batches * global_batch_size
+        dataset = dataset.with_epoch(num_worker_batches)  # each worker is iterating over this
+    else:
+        # last batches are partial, eval is done on single (master) node
+        num_batches = math.ceil(num_samples / args.batch_size)
+
+    dataloader = wds.WebLoader(
+        dataset,
+        batch_size=None,
+        shuffle=False,
+        num_workers=args.workers,
+        persistent_workers=args.workers > 0,
+    )
+
+    # FIXME not clear which approach is better, with_epoch before vs after dataloader?
+    # hoping to resolve via https://github.com/webdataset/webdataset/issues/169
+    # if is_train:
+    #     # roll over and repeat a few samples to get same number of full batches on each node
+    #     global_batch_size = args.batch_size * args.world_size
+    #     num_batches = math.ceil(num_samples / global_batch_size)
+    #     num_workers = max(1, args.workers)
+    #     num_batches = math.ceil(num_batches / num_workers) * num_workers
+    #     num_samples = num_batches * global_batch_size
+    #     dataloader = dataloader.with_epoch(num_batches)
+    # else:
+    #     # last batches are partial, eval is done on single (master) node
+    #     num_batches = math.ceil(num_samples / args.batch_size)
+
+    # add meta-data to dataloader instance for convenience
+    dataloader.num_batches = num_batches
+    dataloader.num_samples = num_samples
+
+    return DataInfo(dataloader=dataloader, shared_epoch=shared_epoch)
+
+
+
+def get_data(args, epoch=0):
+    data = {}
+
+    data["train"] = get_wds_dataset(args, is_train=True, epoch=epoch)
+
+    return data
+
+
+if __name__ == '__main__':
+    args = parse_args(sys.argv[1:])
+    args.workers = 10
+    args.batch_size = 16
+    args.world_size = 1
+    args.num_frames = 8
+    args.clip_type = 'vl'
+    args.model = "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K"
+    args.train_data = '/apdcephfs_cq3/share_1311970/lb/vat2webdata/check_8frm_title_ofa_polishmplug_1tml_1dep/{00000..03020}.tar'
+    args.train_num_samples = 10_000
+    args.dataset_type = 'webdataset'
+
+
+
+    data = get_data(args, epoch=0)
+
+    data['train'].set_epoch(0)  # set epoch in process safe manner via sampler or shared_epoch
+    dataloader = data['train'].dataloader
+    num_batches_per_epoch = dataloader.num_batches // args.accum_freq
+    print(num_batches_per_epoch)
+
+
+    for i, batch in enumerate(tqdm(dataloader)):
+        images, input_ids, attention_mask = batch
+        # print(images.shape, input_ids.shape, attention_mask.shape)
+        # break

RAG/Knowledge_Database/languagebind_main/data/process_audio.py

@@ -0,0 +1,118 @@
+import logging
+
+import numpy as np
+import torch
+import torchaudio
+import torchvision
+from torchvision.transforms import transforms
+from torch.nn import functional as F
+
+torchaudio.set_audio_backend("soundfile")
+
+def torchaudio_loader(path):
+    return torchaudio.load(path)
+
+def int16_to_float32_torch(x):
+    return (x / 32767.0).type(torch.float32)
+
+def float32_to_int16_torch(x):
+    x = torch.clamp(x, min=-1., max=1.)
+    return (x * 32767.).type(torch.int16)
+
+DEFAULT_AUDIO_FRAME_SHIFT_MS = 10
+
+class AudioTransform:
+    def __init__(self, args):
+        self.sample_rate = args.audio_sample_rate
+        self.num_mel_bins = args.num_mel_bins
+        self.target_length = args.target_length
+        self.audio_mean = args.audio_mean
+        self.audio_std = args.audio_std
+        self.mean = []
+        self.std = []
+        # mean=-4.2677393
+        # std=4.5689974
+        # self.norm = transforms.Normalize(mean=self.audio_mean, std=self.audio_std)
+
+
+    def __call__(self, audio_data_and_origin_sr):
+        audio_data, origin_sr = audio_data_and_origin_sr
+        if self.sample_rate != origin_sr:
+            # print(audio_data.shape, origin_sr)
+            audio_data = torchaudio.functional.resample(audio_data, orig_freq=origin_sr, new_freq=self.sample_rate)
+        waveform_melspec = self.waveform2melspec(audio_data)
+        return waveform_melspec
+
+
+    def waveform2melspec(self, audio_data):
+        mel = self.get_mel(audio_data)
+        if mel.shape[0] > self.target_length:
+            # split to three parts
+            chunk_frames = self.target_length
+            total_frames = mel.shape[0]
+            ranges = np.array_split(list(range(0, total_frames - chunk_frames + 1)), 3)
+            # print('total_frames-chunk_frames:', total_frames-chunk_frames,
+            #       'len(audio_data):', len(audio_data),
+            #       'chunk_frames:', chunk_frames,
+            #       'total_frames:', total_frames)
+            if len(ranges[1]) == 0:  # if the audio is too short, we just use the first chunk
+                ranges[1] = [0]
+            if len(ranges[2]) == 0:  # if the audio is too short, we just use the first chunk
+                ranges[2] = [0]
+            # randomly choose index for each part
+            idx_front = np.random.choice(ranges[0])
+            idx_middle = np.random.choice(ranges[1])
+            idx_back = np.random.choice(ranges[2])
+            # idx_front = ranges[0][0]  # fixed
+            # idx_middle = ranges[1][0]
+            # idx_back = ranges[2][0]
+            # select mel
+            mel_chunk_front = mel[idx_front:idx_front + chunk_frames, :]
+            mel_chunk_middle = mel[idx_middle:idx_middle + chunk_frames, :]
+            mel_chunk_back = mel[idx_back:idx_back + chunk_frames, :]
+            # print(total_frames, idx_front, idx_front + chunk_frames, idx_middle, idx_middle + chunk_frames, idx_back, idx_back + chunk_frames)
+            # stack
+            mel_fusion = torch.stack([mel_chunk_front, mel_chunk_middle, mel_chunk_back], dim=0)
+        elif mel.shape[0] < self.target_length:  # padding if too short
+            n_repeat = int(self.target_length / mel.shape[0]) + 1
+            # print(self.target_length, mel.shape[0], n_repeat)
+            mel = mel.repeat(n_repeat, 1)[:self.target_length, :]
+            mel_fusion = torch.stack([mel, mel, mel], dim=0)
+        else:  # if equal
+            mel_fusion = torch.stack([mel, mel, mel], dim=0)
+        mel_fusion = mel_fusion.transpose(1, 2)  # [3, target_length, mel_bins] -> [3, mel_bins, target_length]
+
+        # self.mean.append(mel_fusion.mean())
+        # self.std.append(mel_fusion.std())
+        mel_fusion = (mel_fusion - self.audio_mean) / (self.audio_std * 2)
+        return mel_fusion
+
+    def get_mel(self, audio_data):
+        # mel shape: (n_mels, T)
+        audio_data -= audio_data.mean()
+        mel = torchaudio.compliance.kaldi.fbank(
+            audio_data,
+            htk_compat=True,
+            sample_frequency=self.sample_rate,
+            use_energy=False,
+            window_type="hanning",
+            num_mel_bins=self.num_mel_bins,
+            dither=0.0,
+            frame_length=25,
+            frame_shift=DEFAULT_AUDIO_FRAME_SHIFT_MS,
+        )
+        return mel  # (T, n_mels)
+
+
+
+def get_audio_transform(args):
+    return AudioTransform(args)
+
+def load_and_transform_audio(
+    audio_path,
+    transform,
+):
+    waveform_and_sr = torchaudio_loader(audio_path)
+    audio_outputs = transform(waveform_and_sr)
+
+    return audio_outputs

RAG/Knowledge_Database/languagebind_main/data/process_depth.py

@@ -0,0 +1,55 @@
+import PIL
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+from torch import nn
+from torchvision import transforms
+from open_clip.constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
+
+
+def opencv_loader(path):
+    return cv2.imread(path, cv2.IMREAD_UNCHANGED).astype('float32')
+
+
+class DepthNorm(nn.Module):
+    def __init__(
+        self,
+        max_depth=0,
+        min_depth=0.01,
+    ):
+        super().__init__()
+        self.max_depth = max_depth
+        self.min_depth = min_depth
+        self.scale = 1000.0  # nyuv2 abs.depth
+
+    def forward(self, image):
+        # image = np.array(image)
+        depth_img = image / self.scale  # (H, W)   in meters
+        depth_img = depth_img.clip(min=self.min_depth)
+        if self.max_depth != 0:
+            depth_img = depth_img.clip(max=self.max_depth)
+            depth_img /= self.max_depth   #  0-1
+        else:
+            depth_img /= depth_img.max()
+        depth_img = torch.from_numpy(depth_img).unsqueeze(0).repeat(3, 1, 1)  # assume image
+        return depth_img.to(torch.get_default_dtype())
+
+def get_depth_transform(args):
+    transform = transforms.Compose(
+        [
+            DepthNorm(max_depth=args.max_depth),
+            transforms.Resize(224, interpolation=transforms.InterpolationMode.BICUBIC),
+            transforms.CenterCrop(224),
+            transforms.Normalize(OPENAI_DATASET_MEAN, OPENAI_DATASET_STD),  # assume image
+            # transforms.Normalize((0.5, ), (0.5, ))  # 0-1 to norm distribution
+            # transforms.Normalize((0.0418, ), (0.0295, ))  # sun rgb-d  imagebind
+            # transforms.Normalize((0.02, ), (0.00295, ))  # nyuv2
+        ]
+    )
+    return transform
+
+def load_and_transform_depth(depth_path, transform):
+    depth = opencv_loader(depth_path)
+    depth_outputs = transform(depth)
+    return {'pixel_values': depth_outputs}

RAG/Knowledge_Database/languagebind_main/data/process_image.py

@@ -0,0 +1,25 @@
+from PIL import Image
+
+from open_clip import image_transform, OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
+
+
+def image_loader(path):
+    return Image.open(path)
+
+def get_image_transform(args):
+    preprocess_val = image_transform(
+        args.image_size,
+        is_train=False,
+        mean=OPENAI_DATASET_MEAN,
+        std=OPENAI_DATASET_STD,
+    )
+    return preprocess_val
+
+def load_and_transform_image(
+    image_path,
+    transform,
+):
+    image = image_loader(image_path)
+    image_outputs = transform(image)
+
+    return {'pixel_values': image_outputs}

RAG/Knowledge_Database/languagebind_main/data/process_text.py

@@ -0,0 +1,202 @@
+import os
+
+import torch
+import gzip
+import html
+import io
+from functools import lru_cache
+from typing import List, Tuple
+
+import ftfy
+import regex as re
+from iopath.common.file_io import g_pathmgr
+BPE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
+
+# Modified from github.com/openai/CLIP
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = (
+        list(range(ord("!"), ord("~") + 1))
+        + list(range(ord("¡"), ord("¬") + 1))
+        + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+def basic_clean(text):
+    text = ftfy.fix_text(text)
+    text = html.unescape(html.unescape(text))
+    return text.strip()
+
+
+def whitespace_clean(text):
+    text = re.sub(r"\s+", " ", text)
+    text = text.strip()
+    return text
+
+class SimpleTokenizer(object):
+    def __init__(self, bpe_path: str, context_length=77):
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+
+        with g_pathmgr.open(bpe_path, "rb") as fh:
+            bpe_bytes = io.BytesIO(fh.read())
+            merges: List[str] = gzip.open(bpe_bytes).read().decode("utf-8").split("\n")
+        merges = merges[1 : 49152 - 256 - 2 + 1]
+        merges: List[Tuple[str, ...]] = [tuple(merge.split()) for merge in merges]
+        vocab = list(bytes_to_unicode().values())
+        vocab = vocab + [v + "</w>" for v in vocab]
+        for merge in merges:
+            vocab.append("".join(merge))
+        vocab.extend(["<|startoftext|>", "<|endoftext|>"])
+        self.encoder = dict(zip(vocab, range(len(vocab))))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.bpe_ranks = dict(zip(merges, range(len(merges))))
+        self.cache = {
+            "<|startoftext|>": "<|startoftext|>",
+            "<|endoftext|>": "<|endoftext|>",
+        }
+        self.pat = re.compile(
+            r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""",
+            re.IGNORECASE,
+        )
+        self.context_length = context_length
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token[:-1]) + (token[-1] + "</w>",)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token + "</w>"
+
+        while True:
+            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = " ".join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = "".join(self.byte_encoder[b] for b in token.encode("utf-8"))
+            bpe_tokens.extend(
+                self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ")
+            )
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = "".join([self.decoder[token] for token in tokens])
+        text = (
+            bytearray([self.byte_decoder[c] for c in text])
+            .decode("utf-8", errors="replace")
+            .replace("</w>", " ")
+        )
+        return text
+
+    def __call__(self, texts, context_length=None):
+        if not context_length:
+            context_length = self.context_length
+
+        if isinstance(texts, str):
+            texts = [texts]
+
+        sot_token = self.encoder["<|startoftext|>"]
+        eot_token = self.encoder["<|endoftext|>"]
+        all_tokens = [[sot_token] + self.encode(text) + [eot_token] for text in texts]
+        result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+
+        for i, tokens in enumerate(all_tokens):
+            tokens = tokens[:context_length]
+            result[i, : len(tokens)] = torch.tensor(tokens)
+
+        if len(result) == 1:
+            return result[0]
+        return result
+
+def clean_youtube(text, is_tags=False):
+    text = text.lower() + ' '
+    text = re.sub(
+        r'#video|video|#shorts|shorts| shorts|#short| short|#youtubeshorts|youtubeshorts|#youtube| youtube|#shortsyoutube|#ytshorts|ytshorts|#ytshort|#shortvideo|shortvideo|#shortsfeed|#tiktok|tiktok|#tiktokchallenge|#myfirstshorts|#myfirstshort|#viral|viralvideo|viral|#viralshorts|#virlshort|#ytviralshorts',
+        ' ', text)
+    text = re.sub(r' s |short|youtube|virlshort|#', ' ', text)
+    pattern = r'[^a-zA-Z0-9\s\.,;:?!\'\"|]'
+    if is_tags:
+        pattern = r'[^a-zA-Z0-9\s]'
+    text = re.sub(pattern, '', text)
+    text = whitespace_clean(basic_clean(text))
+    return text
+
+def load_and_transform_text(text, tokenizer, title=True):
+    if title:
+        title_hashtags = text.split('#')
+        title, hashtags = title_hashtags[0], '#' + '#'.join(title_hashtags[1:])
+        title = clean_youtube(title)
+        hashtags = clean_youtube(hashtags, is_tags=True)
+        text = title + ', ' + hashtags
+    if text == '' or text.isspace():
+        raise ValueError('text is empty')
+    input_ids, attention_mask = tokenizer(text)
+    return {'input_ids': input_ids.squeeze(), 'attention_mask': attention_mask.squeeze()}
+
+
+
+if __name__ == '__main__':
+    load_and_transform_text("bpe/bpe_simple_vocab_16e6.txt.gz")

RAG/Knowledge_Database/languagebind_main/data/process_thermal.py

@@ -0,0 +1,26 @@
+import PIL
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+from torch import nn
+from torchvision import transforms
+from open_clip.constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
+
+
+
+def get_thermal_transform(args):
+    transform = transforms.Compose(
+        [
+            transforms.ToTensor(),
+            transforms.Resize(224, interpolation=transforms.InterpolationMode.BICUBIC),
+            transforms.CenterCrop(224),
+            transforms.Normalize(OPENAI_DATASET_MEAN, OPENAI_DATASET_STD)  # assume image
+        ]
+    )
+    return transform
+
+def load_and_transform_thermal(thermal_path, transform):
+    thermal = Image.open(thermal_path)
+    thermal_outputs = transform(thermal)
+    return {'pixel_values': thermal_outputs}

RAG/Knowledge_Database/languagebind_main/data/process_video.py

@@ -0,0 +1,161 @@
+
+import io
+import logging
+import os
+
+import cv2
+import numpy as np
+import torch
+import decord
+import torchvision.transforms
+from PIL import Image
+from decord import VideoReader, cpu
+
+try:
+    from petrel_client.client import Client
+    petrel_backend_imported = True
+except (ImportError, ModuleNotFoundError):
+    petrel_backend_imported = False
+
+
+from pytorchvideo.data.encoded_video import EncodedVideo
+from torchvision.transforms import Compose, Lambda, ToTensor
+from torchvision.transforms._transforms_video import NormalizeVideo, RandomCropVideo, RandomHorizontalFlipVideo
+from pytorchvideo.transforms import ApplyTransformToKey, ShortSideScale, UniformTemporalSubsample
+import sys
+sys.path.append('../')
+from open_clip import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
+from os.path import join as opj
+
+
+def get_video_loader(use_petrel_backend: bool = True,
+                     enable_mc: bool = True,
+                     conf_path: str = None):
+    if petrel_backend_imported and use_petrel_backend:
+        _client = Client(conf_path=conf_path, enable_mc=enable_mc)
+    else:
+        _client = None
+
+    def _loader(video_path):
+        if _client is not None and 's3:' in video_path:
+            video_path = io.BytesIO(_client.get(video_path))
+
+        vr = VideoReader(video_path, num_threads=1, ctx=cpu(0))
+        return vr
+
+    return _loader
+
+
+decord.bridge.set_bridge('torch')
+# video_loader = get_video_loader()
+
+
+def get_video_transform(args):
+    if args.video_decode_backend == 'pytorchvideo':
+        transform = ApplyTransformToKey(
+            key="video",
+            transform=Compose(
+                [
+                    UniformTemporalSubsample(args.num_frames),
+                    Lambda(lambda x: x / 255.0),
+                    NormalizeVideo(mean=OPENAI_DATASET_MEAN, std=OPENAI_DATASET_STD),
+                    ShortSideScale(size=224),
+                    RandomCropVideo(size=224),
+                    RandomHorizontalFlipVideo(p=0.5),
+                ]
+            ),
+        )
+
+    elif args.video_decode_backend == 'decord':
+
+        transform = Compose(
+            [
+                # UniformTemporalSubsample(num_frames),
+                Lambda(lambda x: x / 255.0),
+                NormalizeVideo(mean=OPENAI_DATASET_MEAN, std=OPENAI_DATASET_STD),
+                ShortSideScale(size=224),
+                RandomCropVideo(size=224),
+                RandomHorizontalFlipVideo(p=0.5),
+            ]
+        )
+
+    elif args.video_decode_backend == 'opencv':
+        transform = Compose(
+            [
+                # UniformTemporalSubsample(num_frames),
+                Lambda(lambda x: x / 255.0),
+                NormalizeVideo(mean=OPENAI_DATASET_MEAN, std=OPENAI_DATASET_STD),
+                ShortSideScale(size=224),
+                RandomCropVideo(size=224),
+                RandomHorizontalFlipVideo(p=0.5),
+            ]
+        )
+
+    elif args.video_decode_backend == 'imgs':
+        transform = Compose(
+            [
+                # UniformTemporalSubsample(num_frames),
+                # Lambda(lambda x: x / 255.0),
+                NormalizeVideo(mean=OPENAI_DATASET_MEAN, std=OPENAI_DATASET_STD),
+                ShortSideScale(size=224),
+                RandomCropVideo(size=224),
+                RandomHorizontalFlipVideo(p=0.5),
+            ]
+        )
+    else:
+        raise NameError('video_decode_backend should specify in (pytorchvideo, decord, opencv, imgs)')
+    return transform
+
+def load_and_transform_video(
+    video_path,
+    transform,
+    video_decode_backend='opencv',
+    clip_start_sec=0.0,
+    clip_end_sec=None,
+    num_frames=8,
+):
+    if video_decode_backend == 'pytorchvideo':
+        #  decord pyav
+        video = EncodedVideo.from_path(video_path, decoder="decord", decode_audio=False)
+        duration = video.duration
+        start_sec = clip_start_sec  # secs
+        end_sec = clip_end_sec if clip_end_sec is not None else duration  # secs
+        video_data = video.get_clip(start_sec=start_sec, end_sec=end_sec)
+        video_outputs = transform(video_data)
+
+    elif video_decode_backend == 'decord':
+        decord_vr = VideoReader(video_path, ctx=cpu(0))
+        duration = len(decord_vr)
+        frame_id_list = np.linspace(0, duration-1, num_frames, dtype=int)
+        video_data = decord_vr.get_batch(frame_id_list)
+        video_data = video_data.permute(3, 0, 1, 2)  # (T, H, W, C) -> (C, T, H, W)
+        video_outputs = transform(video_data)
+
+    elif video_decode_backend == 'opencv':
+        cv2_vr = cv2.VideoCapture(video_path)
+        duration = int(cv2_vr.get(cv2.CAP_PROP_FRAME_COUNT))
+        frame_id_list = np.linspace(0, duration-1, num_frames, dtype=int)
+
+        video_data = []
+        for frame_idx in frame_id_list:
+            cv2_vr.set(1, frame_idx)
+            _, frame = cv2_vr.read()
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            video_data.append(torch.from_numpy(frame).permute(2, 0, 1))
+        cv2_vr.release()
+        video_data = torch.stack(video_data, dim=1)
+        video_outputs = transform(video_data)
+
+    elif video_decode_backend == 'imgs':
+        resize256_folder = video_path.replace('.mp4', '_resize256_folder')
+        video_data = [ToTensor()(Image.open(opj(resize256_folder, f'{i}.jpg'))) for i in range(8)]
+        video_data = torch.stack(video_data, dim=1)
+        # print(video_data.shape, video_data.max(), video_data.min())
+        video_outputs = transform(video_data)
+
+    else:
+        raise NameError('video_decode_backend should specify in (pytorchvideo, decord, opencv, imgs)')
+    return {'pixel_values': video_outputs}
+
+if __name__ == '__main__':
+    load_and_transform_video(r"D:\ONE-PEACE-main\lb_test\zHSOYcZblvY.mp4")

RAG/Knowledge_Database/languagebind_main/gradio_app.py

@@ -0,0 +1,219 @@
+import sys
+
+import gradio as gr
+import argparse
+import numpy as np
+import torch
+from torch import nn
+
+from languagebind import LanguageBind, transform_dict, LanguageBindImageTokenizer, to_device
+
+code_highlight_css = (
+"""
+#chatbot .hll { background-color: #ffffcc }
+#chatbot .c { color: #408080; font-style: italic }
+#chatbot .err { border: 1px solid #FF0000 }
+#chatbot .k { color: #008000; font-weight: bold }
+#chatbot .o { color: #666666 }
+#chatbot .ch { color: #408080; font-style: italic }
+#chatbot .cm { color: #408080; font-style: italic }
+#chatbot .cp { color: #BC7A00 }
+#chatbot .cpf { color: #408080; font-style: italic }
+#chatbot .c1 { color: #408080; font-style: italic }
+#chatbot .cs { color: #408080; font-style: italic }
+#chatbot .gd { color: #A00000 }
+#chatbot .ge { font-style: italic }
+#chatbot .gr { color: #FF0000 }
+#chatbot .gh { color: #000080; font-weight: bold }
+#chatbot .gi { color: #00A000 }
+#chatbot .go { color: #888888 }
+#chatbot .gp { color: #000080; font-weight: bold }
+#chatbot .gs { font-weight: bold }
+#chatbot .gu { color: #800080; font-weight: bold }
+#chatbot .gt { color: #0044DD }
+#chatbot .kc { color: #008000; font-weight: bold }
+#chatbot .kd { color: #008000; font-weight: bold }
+#chatbot .kn { color: #008000; font-weight: bold }
+#chatbot .kp { color: #008000 }
+#chatbot .kr { color: #008000; font-weight: bold }
+#chatbot .kt { color: #B00040 }
+#chatbot .m { color: #666666 }
+#chatbot .s { color: #BA2121 }
+#chatbot .na { color: #7D9029 }
+#chatbot .nb { color: #008000 }
+#chatbot .nc { color: #0000FF; font-weight: bold }
+#chatbot .no { color: #880000 }
+#chatbot .nd { color: #AA22FF }
+#chatbot .ni { color: #999999; font-weight: bold }
+#chatbot .ne { color: #D2413A; font-weight: bold }
+#chatbot .nf { color: #0000FF }
+#chatbot .nl { color: #A0A000 }
+#chatbot .nn { color: #0000FF; font-weight: bold }
+#chatbot .nt { color: #008000; font-weight: bold }
+#chatbot .nv { color: #19177C }
+#chatbot .ow { color: #AA22FF; font-weight: bold }
+#chatbot .w { color: #bbbbbb }
+#chatbot .mb { color: #666666 }
+#chatbot .mf { color: #666666 }
+#chatbot .mh { color: #666666 }
+#chatbot .mi { color: #666666 }
+#chatbot .mo { color: #666666 }
+#chatbot .sa { color: #BA2121 }
+#chatbot .sb { color: #BA2121 }
+#chatbot .sc { color: #BA2121 }
+#chatbot .dl { color: #BA2121 }
+#chatbot .sd { color: #BA2121; font-style: italic }
+#chatbot .s2 { color: #BA2121 }
+#chatbot .se { color: #BB6622; font-weight: bold }
+#chatbot .sh { color: #BA2121 }
+#chatbot .si { color: #BB6688; font-weight: bold }
+#chatbot .sx { color: #008000 }
+#chatbot .sr { color: #BB6688 }
+#chatbot .s1 { color: #BA2121 }
+#chatbot .ss { color: #19177C }
+#chatbot .bp { color: #008000 }
+#chatbot .fm { color: #0000FF }
+#chatbot .vc { color: #19177C }
+#chatbot .vg { color: #19177C }
+#chatbot .vi { color: #19177C }
+#chatbot .vm { color: #19177C }
+#chatbot .il { color: #666666 }
+""")
+#.highlight  { background: #f8f8f8; }
+
+title_markdown = ("""
+<div style="display: flex; justify-content: center;">
+  <a href="https://github.com/PKU-YuanGroup/LanguageBind">
+    <img src="https://z1.ax1x.com/2023/10/16/piCuiDS.png" alt="LanguageBind🚀" border="0" style="height: 200px; margin-right: 20px;">
+  </a>
+  <a href="https://github.com/PKU-YuanGroup/LanguageBind">
+    <img src="https://z1.ax1x.com/2023/11/04/piMLoQ0.png" style="height: 200px;">
+  </a>
+</div>
+<h2 align="center"> LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment </h2>
+
+<h5 align="center"> If you like our project, please give us a star ✨ on Github for latest update.  </h2>
+
+<div align="center">
+    <div style="display:flex; gap: 0.25rem;" align="center">
+        <a href='https://github.com/PKU-YuanGroup/LanguageBind'><img src='https://img.shields.io/badge/Github-Code-blue'></a>
+        <a href="https://arxiv.org/pdf/2310.01852.pdf"><img src="https://img.shields.io/badge/Arxiv-2310.01852-red"></a>
+        <a href='https://github.com/PKU-YuanGroup/LanguageBind/stargazers'><img src='https://img.shields.io/github/stars/PKU-YuanGroup/LanguageBind.svg?style=social'></a>
+    </div>
+</div>
+""")
+css = code_highlight_css + """
+pre {
+    white-space: pre-wrap;       /* Since CSS 2.1 */
+    white-space: -moz-pre-wrap;  /* Mozilla, since 1999 */
+    white-space: -pre-wrap;      /* Opera 4-6 */
+    white-space: -o-pre-wrap;    /* Opera 7 */
+    word-wrap: break-word;       /* Internet Explorer 5.5+ */
+}
+"""
+
+
+def image_to_language(image, language):
+    inputs = {}
+    inputs['image'] = to_device(modality_transform['image'](image), device)
+    inputs['language'] = to_device(modality_transform['language'](language, max_length=77, padding='max_length',
+                                                                  truncation=True, return_tensors='pt'), device)
+    with torch.no_grad():
+        embeddings = model(inputs)
+    return (embeddings['image'] @ embeddings['language'].T).item()
+
+
+def video_to_language(video, language):
+    inputs = {}
+    inputs['video'] = to_device(modality_transform['video'](video), device)
+    inputs['language'] = to_device(modality_transform['language'](language, max_length=77, padding='max_length',
+                                                                  truncation=True, return_tensors='pt'), device)
+    with torch.no_grad():
+        embeddings = model(inputs)
+    return (embeddings['video'] @ embeddings['language'].T).item()
+
+
+def audio_to_language(audio, language):
+    inputs = {}
+    inputs['audio'] = to_device(modality_transform['audio'](audio), device)
+    inputs['language'] = to_device(modality_transform['language'](language, max_length=77, padding='max_length',
+                                                                  truncation=True, return_tensors='pt'), device)
+    with torch.no_grad():
+        embeddings = model(inputs)
+    return (embeddings['audio'] @ embeddings['language'].T).item()
+
+
+def depth_to_language(depth, language):
+    inputs = {}
+    inputs['depth'] = to_device(modality_transform['depth'](depth.name), device)
+    inputs['language'] = to_device(modality_transform['language'](language, max_length=77, padding='max_length',
+                                                                  truncation=True, return_tensors='pt'), device)
+    with torch.no_grad():
+        embeddings = model(inputs)
+    return (embeddings['depth'] @ embeddings['language'].T).item()
+
+
+def thermal_to_language(thermal, language):
+    inputs = {}
+    inputs['thermal'] = to_device(modality_transform['thermal'](thermal), device)
+    inputs['language'] = to_device(modality_transform['language'](language, max_length=77, padding='max_length',
+                                                                  truncation=True, return_tensors='pt'), device)
+    with torch.no_grad():
+        embeddings = model(inputs)
+    return (embeddings['thermal'] @ embeddings['language'].T).item()
+
+if __name__ == '__main__':
+    device = 'cuda:0'
+    device = torch.device(device)
+    clip_type = {
+        'video': 'LanguageBind_Video_FT',  # also LanguageBind_Video
+        'audio': 'LanguageBind_Audio_FT',  # also LanguageBind_Audio
+        'thermal': 'LanguageBind_Thermal',
+        'image': 'LanguageBind_Image',
+        'depth': 'LanguageBind_Depth',
+    }
+    model = LanguageBind(clip_type=clip_type, use_temp=False)
+    model = model.to(device)
+    model.eval()
+    pretrained_ckpt = f'lb203/LanguageBind_Image'
+    tokenizer = LanguageBindImageTokenizer.from_pretrained(pretrained_ckpt, cache_dir='./cache_dir/tokenizer_cache_dir')
+    modality_transform = {c: transform_dict[c](model.modality_config[c]) for c in clip_type}
+    modality_transform['language'] = tokenizer
+
+    with gr.Blocks(title="LanguageBind🚀", css=css) as demo:
+        gr.Markdown(title_markdown)
+        with gr.Row():
+            with gr.Column():
+                image = gr.Image(type="filepath", height=224, width=224, label='Image Input')
+                language_i = gr.Textbox(lines=2, label='Text Input')
+                out_i = gr.Textbox(label='Similarity of Image to Text')
+                b_i = gr.Button("Calculate similarity of Image to Text")
+            with gr.Column():
+                video = gr.Video(type="filepath", height=224, width=224, label='Video Input')
+                language_v = gr.Textbox(lines=2, label='Text Input')
+                out_v = gr.Textbox(label='Similarity of Video to Text')
+                b_v = gr.Button("Calculate similarity of Video to Text")
+            with gr.Column():
+                audio = gr.Audio(type="filepath", label='Audio Input')
+                language_a = gr.Textbox(lines=2, label='Text Input')
+                out_a = gr.Textbox(label='Similarity of Audio to Text')
+                b_a = gr.Button("Calculate similarity of Audio to Text")
+        with gr.Row():
+            with gr.Column():
+                depth = gr.File(height=224, width=224, label='Depth Input, need a .png file, 16 bit, with values ranging from 0-10000 (representing 0-10 metres, but 1000 times)')
+                language_d = gr.Textbox(lines=2, label='Text Input')
+                out_d = gr.Textbox(label='Similarity of Depth to Text')
+                b_d = gr.Button("Calculate similarity of Depth to Text")
+            with gr.Column():
+                thermal = gr.Image(type="filepath", height=224, width=224, label='Thermal Input, you should first convert to RGB')
+                language_t = gr.Textbox(lines=2, label='Text Input')
+                out_t = gr.Textbox(label='Similarity of Thermal to Text')
+                b_t = gr.Button("Calculate similarity of Thermal to Text")
+
+        b_i.click(image_to_language, inputs=[image, language_i], outputs=out_i)
+        b_a.click(audio_to_language, inputs=[audio, language_a], outputs=out_a)
+        b_v.click(video_to_language, inputs=[video, language_v], outputs=out_v)
+        b_d.click(depth_to_language, inputs=[depth, language_d], outputs=out_d)
+        b_t.click(thermal_to_language, inputs=[thermal, language_t], outputs=out_t)
+
+    demo.launch()

RAG/Knowledge_Database/languagebind_main/i_cls/datasets.py

@@ -0,0 +1,31 @@
+import torch
+
+from data.build_datasets import DataInfo
+from open_clip import image_transform, OPENAI_DATASET_STD, OPENAI_DATASET_MEAN, get_tokenizer
+from torchvision import datasets
+
+
+def get_imagenet(args, split):
+    assert split in ["val", "v2"]
+    preprocess_val = image_transform(
+        args.image_size,
+        is_train=False,
+        mean=OPENAI_DATASET_MEAN,
+        std=OPENAI_DATASET_STD,
+    )
+    if split == "v2":
+        from imagenetv2_pytorch import ImageNetV2Dataset
+        dataset = ImageNetV2Dataset(location=args.imagenet_v2, transform=preprocess_val)
+    else:
+        data_path = args.imagenet_val
+        assert data_path
+        dataset = datasets.ImageFolder(data_path, transform=preprocess_val)
+
+    dataloader = torch.utils.data.DataLoader(
+        dataset,
+        batch_size=args.batch_size,
+        num_workers=args.workers,
+        sampler=None,
+    )
+
+    return DataInfo(dataloader=dataloader, sampler=None)