psuhing to git

.env.example

@@ -0,0 +1,25 @@
+PROJECT_NAME=KnowledgeHub
+VERSION=1.0.0
+API_V1_STR=/api/v1
+
+QDRANT_URL=http://localhost:6333
+QDRANT_API_KEY=
+QDRANT_COLLECTION_NAME=knowledge_base
+
+NEMOTRON_PARSE_MODEL=Qwen/Qwen2-VL-2B-Instruct
+NEMOTRON_EMBED_MODEL=nvidia/llama-nemotron-colembed-vl-3b-v2
+EMBEDDING_DEVICE=cpu
+HF_TOKEN=
+NVIDIA_API_KEY=
+NVIDIA_CHAT_MODEL=nvidia/nvidia-nemotron-nano-9b-v2
+NVIDIA_API_URL=https://integrate.api.nvidia.com/v1
+
+MAX_UPLOAD_SIZE=10485760
+CHUNK_SIZE=1100
+CHUNK_OVERLAP=180
+ZEROGPU_DURATION_SECONDS=180
+CHAT_TEMPERATURE=0.6
+CHAT_TOP_P=0.95
+CHAT_MAX_TOKENS=2048
+MIN_THINKING_TOKENS=1024
+MAX_THINKING_TOKENS=2048

.gitignore

@@ -0,0 +1,7 @@
+.env
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+data/exports/

README.md

@@ -0,0 +1,58 @@
+# KnowledgeHub Ingestor
+
+KnowledgeHub Ingestor is a modular Gradio app for loading knowledge from:
+
+- YouTube links with public transcripts/captions
+- arXiv links or IDs
+- PDF documents
+
+It extracts text, chunks it, embeds chunks locally with your embedding model, and uploads vectors into Qdrant for retrieval. The answer generation step uses NVIDIA's OpenAI-compatible chat API.
+
+## Setup
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+cp .env.example .env
+```
+
+Add `NVIDIA_API_KEY` to `.env` for chat completions. Start Qdrant locally or point `QDRANT_URL` to your hosted instance.
+
+The default model split is:
+
+- Local parsing model: `Qwen/Qwen2-VL-2B-Instruct`
+- Local embedding model: `nvidia/llama-nemotron-colembed-vl-3b-v2`
+- NVIDIA API chat model: `nvidia/nvidia-nemotron-nano-9b-v2`
+
+## Run
+
+```bash
+python app.py
+```
+
+Open the local Gradio URL printed in the terminal, usually `http://127.0.0.1:7860`.
+
+The app binds to `0.0.0.0:7860`, which is suitable for Hugging Face Spaces and container deployments.
+
+For Hugging Face ZeroGPU Spaces, set:
+
+```bash
+ENABLE_ZEROGPU=true
+EMBEDDING_DEVICE=cuda
+```
+
+The Gradio ingest/search/answer callbacks are decorated with `spaces.GPU` when running on Spaces. Locally, the decorator becomes a no-op.
+
+## Project Structure
+
+```text
+app/
+  core/        settings and shared models
+  extractors/  PDF, arXiv, and YouTube extraction
+  services/    chunking, embeddings, Qdrant, retrieval, ingestion orchestration
+  ui/          Gradio Blocks UI
+  utils/       source detection helpers
+```
+
+YouTube extraction requires captions/transcripts to be available for the video. arXiv ingestion downloads the paper PDF and parses it with `pypdf`.

app.py

@@ -0,0 +1,28 @@
+import logging
+import shutil
+import sys
+from pathlib import Path
+
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+def _clear_app_bytecode() -> None:
+    sys.dont_write_bytecode = True
+    for cache_dir in Path(__file__).resolve().parent.joinpath("app").rglob("__pycache__"):
+        shutil.rmtree(cache_dir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    logger.info("Preparing BuildSmall app")
+    _clear_app_bytecode()
+    logger.info("Loading Gradio UI")
+    from app.ui.gradio_app import serve
+
+    logger.info("Launching BuildSmall app")
+    serve()
+    logger.info("BuildSmall app stopped")

app/__init__.py

@@ -0,0 +1 @@
+"""KnowledgeHub document ingestion package."""

app/core/__init__.py

@@ -0,0 +1 @@
+"""Core application settings and data models."""

app/core/config.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    PROJECT_NAME: str = "KnowledgeHub"
+    VERSION: str = "1.0.0"
+    API_V1_STR: str = "/api/v1"
+
+    QDRANT_URL: str = "http://localhost:6333"
+    QDRANT_API_KEY: str = ""
+    QDRANT_COLLECTION_NAME: str = "knowledge_base"
+
+    NEMOTRON_PARSE_MODEL: str = "Qwen/Qwen2-VL-2B-Instruct"
+    NEMOTRON_EMBED_MODEL: str = "nvidia/llama-nemotron-colembed-vl-3b-v2"
+    EMBEDDING_DEVICE: str = "cpu"
+    HF_TOKEN: str = ""
+    NVIDIA_API_KEY: str = ""
+    NVIDIA_CHAT_MODEL: str = "nvidia/nvidia-nemotron-nano-9b-v2"
+    NVIDIA_API_URL: str = "https://integrate.api.nvidia.com/v1"
+
+    MAX_UPLOAD_SIZE: int = 10 * 1024 * 1024
+    CHUNK_SIZE: int = 1100
+    CHUNK_OVERLAP: int = 180
+    ZEROGPU_DURATION_SECONDS: int = 180
+    CHAT_TEMPERATURE: float = 0.6
+    CHAT_TOP_P: float = 0.95
+    CHAT_MAX_TOKENS: int = 2048
+    MIN_THINKING_TOKENS: int = 1024
+    MAX_THINKING_TOKENS: int = 2048
+
+    model_config = SettingsConfigDict(
+        env_file=Path(__file__).resolve().parents[2] / ".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    def get_qdrant_url(self) -> str:
+        if self.QDRANT_URL.startswith("https://") and ":" not in self.QDRANT_URL[8:]:
+            return f"{self.QDRANT_URL}:443"
+        return self.QDRANT_URL
+
+
+settings = Settings()

app/core/models.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+
+class SourceType(str, Enum):
+    PDF = "pdf"
+    ARXIV = "arxiv"
+    YOUTUBE = "youtube"
+
+
+@dataclass(frozen=True)
+class Document:
+    source_type: SourceType
+    title: str
+    text: str
+    source: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class Chunk:
+    id: str
+    text: str
+    index: int
+    source_type: SourceType
+    source: str
+    title: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class IngestionResult:
+    document: Document
+    chunks: list[Chunk]
+    collection_name: str
+    export_path: Path
+
+
+@dataclass(frozen=True)
+class SearchResult:
+    score: float
+    text: str
+    title: str
+    source: str
+    source_type: str
+    metadata: dict[str, Any]

app/extractors/__init__.py

@@ -0,0 +1 @@
+"""Source-specific document extractors."""

app/extractors/arxiv.py

@@ -0,0 +1,46 @@
+import tempfile
+from pathlib import Path
+
+import arxiv
+import requests
+
+from app.core.models import Document, SourceType
+from app.extractors.pdf import extract_pdf
+from app.utils.source_detection import extract_arxiv_id
+
+
+def extract_arxiv(value: str) -> Document:
+    paper_id = extract_arxiv_id(value)
+    client = arxiv.Client()
+    search = arxiv.Search(id_list=[paper_id])
+    paper = next(client.results(search), None)
+    if paper is None:
+        raise ValueError(f"No arXiv paper found for {paper_id}.")
+
+    with tempfile.TemporaryDirectory(prefix="knowledgehub_arxiv_") as tmpdir:
+        pdf_url = paper.pdf_url or f"https://arxiv.org/pdf/{paper_id}.pdf"
+        pdf_path = Path(tmpdir) / f"{paper_id}.pdf"
+        response = requests.get(pdf_url, timeout=60)
+        response.raise_for_status()
+        pdf_path.write_bytes(response.content)
+
+        document = extract_pdf(
+            pdf_path,
+            title=paper.title,
+            metadata={
+                "arxiv_id": paper_id,
+                "authors": [str(author) for author in paper.authors],
+                "published": paper.published.isoformat() if paper.published else None,
+                "summary": paper.summary,
+                "pdf_url": pdf_url,
+                "entry_id": paper.entry_id,
+            },
+        )
+
+    return Document(
+        source_type=SourceType.ARXIV,
+        title=document.title,
+        text=document.text,
+        source=paper.entry_id,
+        metadata=document.metadata,
+    )

app/extractors/pdf.py

@@ -0,0 +1,30 @@
+from pathlib import Path
+
+from pypdf import PdfReader
+
+from app.core.models import Document, SourceType
+
+
+def extract_pdf(path: str | Path, title: str | None = None, metadata: dict | None = None) -> Document:
+    pdf_path = Path(path)
+    if not pdf_path.exists():
+        raise FileNotFoundError(f"PDF not found: {pdf_path}")
+
+    reader = PdfReader(str(pdf_path))
+    pages: list[str] = []
+    for page_number, page in enumerate(reader.pages, start=1):
+        text = page.extract_text() or ""
+        if text.strip():
+            pages.append(f"\n\n[Page {page_number}]\n{text.strip()}")
+
+    combined_text = "\n".join(pages).strip()
+    if not combined_text:
+        raise ValueError("No selectable text was found in this PDF.")
+
+    return Document(
+        source_type=SourceType.PDF,
+        title=title or pdf_path.stem,
+        text=combined_text,
+        source=str(pdf_path),
+        metadata={"pages": len(reader.pages), **(metadata or {})},
+    )

app/extractors/youtube.py

@@ -0,0 +1,47 @@
+from urllib.parse import parse_qs, urlparse
+
+from youtube_transcript_api import YouTubeTranscriptApi
+
+from app.core.models import Document, SourceType
+
+
+def _extract_video_id(url: str) -> str:
+    parsed = urlparse(url.strip())
+    if parsed.netloc.endswith("youtu.be"):
+        return parsed.path.strip("/")
+    if "youtube.com" in parsed.netloc:
+        query = parse_qs(parsed.query)
+        if "v" in query:
+            return query["v"][0]
+        if parsed.path.startswith("/shorts/"):
+            return parsed.path.split("/")[2]
+    raise ValueError("Could not find a YouTube video ID in the URL.")
+
+
+def extract_youtube(url: str) -> Document:
+    video_id = _extract_video_id(url)
+    api = YouTubeTranscriptApi()
+    if hasattr(api, "fetch"):
+        transcript = api.fetch(video_id)
+        transcript_items = transcript.to_raw_data()
+    else:
+        transcript_items = YouTubeTranscriptApi.get_transcript(video_id)
+
+    if not transcript_items:
+        raise ValueError("No transcript was available for this YouTube video.")
+
+    lines = []
+    for item in transcript_items:
+        timestamp = int(item.get("start", 0))
+        minutes, seconds = divmod(timestamp, 60)
+        text = item.get("text", "").strip()
+        if text:
+            lines.append(f"[{minutes:02d}:{seconds:02d}] {text}")
+
+    return Document(
+        source_type=SourceType.YOUTUBE,
+        title=f"YouTube Transcript {video_id}",
+        text="\n".join(lines).strip(),
+        source=url,
+        metadata={"video_id": video_id, "segments": len(transcript_items)},
+    )

app/services/__init__.py

@@ -0,0 +1 @@
+"""Application service layer."""

app/services/chat.py

@@ -0,0 +1,68 @@
+from dataclasses import dataclass
+
+from openai import OpenAI
+
+from app.core.config import settings
+from app.core.models import SearchResult
+
+
+@dataclass(frozen=True)
+class ChatAnswer:
+    answer: str
+    reasoning: str | None
+    context: list[SearchResult]
+
+
+class NvidiaChatClient:
+    def __init__(self):
+        if not settings.NVIDIA_API_KEY:
+            raise ValueError("NVIDIA_API_KEY is required for NVIDIA chat completions.")
+
+        self.client = OpenAI(
+            base_url=settings.NVIDIA_API_URL,
+            api_key=settings.NVIDIA_API_KEY,
+        )
+
+    def answer_with_context(self, question: str, context: list[SearchResult]) -> ChatAnswer:
+        context_text = "\n\n".join(
+            [
+                (
+                    f"[{index}] title={item.title}\n"
+                    f"source={item.source}\n"
+                    f"score={item.score:.4f}\n"
+                    f"text={item.text}"
+                )
+                for index, item in enumerate(context, start=1)
+            ]
+        )
+        messages = [
+            {
+                "role": "system",
+                "content": (
+                    "You are KnowledgeHub's retrieval assistant. Answer only from the "
+                    "provided context. If the context is insufficient, say what is missing. "
+                    "Cite sources using bracket numbers like [1], [2]."
+                ),
+            },
+            {
+                "role": "user",
+                "content": f"Question:\n{question}\n\nRetrieved context:\n{context_text}",
+            },
+        ]
+        completion = self.client.chat.completions.create(
+            model=settings.NVIDIA_CHAT_MODEL,
+            messages=messages,
+            temperature=settings.CHAT_TEMPERATURE,
+            top_p=settings.CHAT_TOP_P,
+            max_tokens=settings.CHAT_MAX_TOKENS,
+            frequency_penalty=0,
+            presence_penalty=0,
+            stream=False,
+            extra_body={
+                "min_thinking_tokens": settings.MIN_THINKING_TOKENS,
+                "max_thinking_tokens": settings.MAX_THINKING_TOKENS,
+            },
+        )
+        message = completion.choices[0].message
+        reasoning = getattr(message, "reasoning_content", None)
+        return ChatAnswer(answer=message.content or "", reasoning=reasoning, context=context)

app/services/chunking.py

@@ -0,0 +1,47 @@
+import re
+import uuid
+
+from app.core.models import Chunk, Document
+
+
+def chunk_document(document: Document, chunk_size: int, overlap: int) -> list[Chunk]:
+    if overlap >= chunk_size:
+        raise ValueError("Chunk overlap must be smaller than chunk size.")
+
+    normalized = re.sub(r"\n{3,}", "\n\n", document.text).strip()
+    if not normalized:
+        raise ValueError("Document is empty after extraction.")
+
+    chunks: list[Chunk] = []
+    start = 0
+    index = 0
+    while start < len(normalized):
+        end = min(start + chunk_size, len(normalized))
+        if end < len(normalized):
+            paragraph_break = normalized.rfind("\n\n", start, end)
+            sentence_break = normalized.rfind(". ", start, end)
+            best_break = max(paragraph_break, sentence_break)
+            if best_break > start + chunk_size // 2:
+                end = best_break + 1
+
+        text = normalized[start:end].strip()
+        if text:
+            digest = str(uuid.uuid5(uuid.NAMESPACE_URL, f"{document.source}:{index}:{text[:80]}"))
+            chunks.append(
+                Chunk(
+                    id=digest,
+                    text=text,
+                    index=index,
+                    source_type=document.source_type,
+                    source=document.source,
+                    title=document.title,
+                    metadata=document.metadata,
+                )
+            )
+            index += 1
+
+        if end == len(normalized):
+            break
+        start = max(0, end - overlap)
+
+    return chunks

app/services/embeddings.py

@@ -0,0 +1,109 @@
+from functools import cached_property, lru_cache
+
+from app.core.config import settings
+from app.utils.zerogpu import is_enabled as zerogpu_is_enabled
+
+
+class LocalEmbeddingClient:
+    def __init__(self, model: str | None = None, device: str | None = None):
+        self.model_name = model or settings.NEMOTRON_EMBED_MODEL
+        self.device = device or _resolve_device()
+
+    @cached_property
+    def model(self):
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError as exc:
+            raise ImportError(
+                "sentence-transformers is required for local embeddings. "
+                "Install dependencies with `pip install -r requirements.txt`."
+            ) from exc
+
+        return SentenceTransformer(
+            self.model_name,
+            device=self.device,
+            token=settings.HF_TOKEN or None,
+            trust_remote_code=True,
+        )
+
+    @cached_property
+    def native_model(self):
+        try:
+            from transformers import AutoModel
+        except ImportError as exc:
+            raise ImportError(
+                "transformers is required for native local embeddings. "
+                "Install dependencies with `pip install -r requirements.txt`."
+            ) from exc
+
+        model = AutoModel.from_pretrained(
+            self.model_name,
+            token=settings.HF_TOKEN or None,
+            trust_remote_code=True,
+            dtype="auto" if self.device != "cpu" else None,
+        )
+        if self.device:
+            model = model.to(self.device)
+        return model.eval()
+
+    def embed_texts(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+
+        try:
+            embeddings = self.model.encode(
+                texts,
+                batch_size=8,
+                normalize_embeddings=True,
+                show_progress_bar=False,
+            )
+            return embeddings.tolist()
+        except ValueError as exc:
+            if "Modality 'text' is not supported" not in str(exc):
+                raise
+
+        embeddings = self._embed_with_native_query_encoder(texts)
+        return embeddings.tolist()
+
+    def _embed_with_native_query_encoder(self, texts: list[str]):
+        try:
+            import torch
+            import torch.nn.functional as F
+        except ImportError as exc:
+            raise ImportError(
+                "torch is required for the native Nemotron embedding path. "
+                "Install dependencies with `pip install -r requirements.txt`."
+            ) from exc
+
+        if not hasattr(self.native_model, "forward_queries"):
+            raise ValueError(
+                f"{self.model_name} does not support SentenceTransformer text encoding "
+                "or a native forward_queries API."
+            )
+
+        with torch.no_grad():
+            output = self.native_model.forward_queries(texts, batch_size=4)
+
+        if isinstance(output, (list, tuple)):
+            output = output[0]
+
+        if not torch.is_tensor(output):
+            output = torch.as_tensor(output)
+
+        if output.ndim == 3:
+            output = output.float().mean(dim=1)
+        elif output.ndim != 2:
+            raise ValueError(f"Unexpected embedding shape from {self.model_name}: {tuple(output.shape)}")
+
+        return F.normalize(output.float(), p=2, dim=1).cpu()
+
+
+@lru_cache(maxsize=1)
+def get_embedding_client() -> LocalEmbeddingClient:
+    return LocalEmbeddingClient()
+
+
+def _resolve_device() -> str:
+    if zerogpu_is_enabled() and settings.EMBEDDING_DEVICE == "cpu":
+        return "cuda"
+    return settings.EMBEDDING_DEVICE

app/services/ingestion.py

@@ -0,0 +1,89 @@
+from pathlib import Path
+
+from app.core.config import settings
+from app.core.models import Document, IngestionResult, SourceType
+from app.extractors.arxiv import extract_arxiv
+from app.extractors.pdf import extract_pdf
+from app.extractors.youtube import extract_youtube
+from app.services.chat import NvidiaChatClient
+from app.services.chunking import chunk_document
+from app.services.embeddings import get_embedding_client
+from app.services.vector_store import QdrantVectorStore
+from app.utils.source_detection import detect_source
+
+
+EXPORT_DIR = Path("data/exports")
+
+
+def extract_document(url: str | None = None, pdf_path: str | None = None) -> Document:
+    source_type = detect_source(url, pdf_path)
+    if source_type == SourceType.PDF:
+        return extract_pdf(str(pdf_path))
+    if source_type == SourceType.ARXIV:
+        return extract_arxiv(str(url))
+    if source_type == SourceType.YOUTUBE:
+        return extract_youtube(str(url))
+    raise ValueError(f"Unsupported source type: {source_type}")
+
+
+def save_markdown(document: Document, chunks_count: int) -> Path:
+    EXPORT_DIR.mkdir(parents=True, exist_ok=True)
+    safe_title = "".join(char if char.isalnum() or char in "-_" else "_" for char in document.title)[:80]
+    path = EXPORT_DIR / f"{safe_title or document.source_type.value}.md"
+    metadata_lines = "\n".join(f"- {key}: {value}" for key, value in document.metadata.items())
+    path.write_text(
+        "\n".join(
+            [
+                f"# {document.title}",
+                "",
+                f"- Source type: {document.source_type.value}",
+                f"- Source: {document.source}",
+                f"- Chunks uploaded: {chunks_count}",
+                metadata_lines,
+                "",
+                "## Extracted Text",
+                "",
+                document.text,
+            ]
+        ),
+        encoding="utf-8",
+    )
+    return path
+
+
+def ingest_source(
+    url: str | None,
+    pdf_path: str | None,
+    chunk_size: int | None = None,
+    chunk_overlap: int | None = None,
+    collection_name: str | None = None,
+) -> IngestionResult:
+    document = extract_document(url=url, pdf_path=pdf_path)
+    chunks = chunk_document(
+        document,
+        chunk_size=chunk_size or settings.CHUNK_SIZE,
+        overlap=chunk_overlap or settings.CHUNK_OVERLAP,
+    )
+    embeddings = get_embedding_client().embed_texts([chunk.text for chunk in chunks])
+    store = QdrantVectorStore(collection_name=collection_name)
+    store.upsert_chunks(chunks, embeddings)
+    export_path = save_markdown(document, len(chunks))
+    return IngestionResult(
+        document=document,
+        chunks=chunks,
+        collection_name=store.collection_name,
+        export_path=export_path,
+    )
+
+
+def search_knowledge_base(query: str, limit: int = 5, collection_name: str | None = None):
+    query_text = query.strip()
+    if not query_text:
+        raise ValueError("Enter a query to search.")
+    embedding = get_embedding_client().embed_texts([query_text])[0]
+    return QdrantVectorStore(collection_name=collection_name).search(embedding, limit=limit)
+
+
+def answer_from_knowledge_base(query: str, limit: int = 5, collection_name: str | None = None):
+    results = search_knowledge_base(query, limit=limit, collection_name=collection_name)
+    return NvidiaChatClient().answer_with_context(query, results)

app/services/vector_store.py

@@ -0,0 +1,80 @@
+from qdrant_client import QdrantClient
+from qdrant_client.http.models import Distance, PointStruct, VectorParams
+
+from app.core.config import settings
+from app.core.models import Chunk, SearchResult
+
+
+class QdrantVectorStore:
+    def __init__(self, collection_name: str | None = None):
+        self.collection_name = collection_name or settings.QDRANT_COLLECTION_NAME
+        self.client = QdrantClient(
+            url=settings.get_qdrant_url(),
+            api_key=settings.QDRANT_API_KEY or None,
+            timeout=60,
+        )
+
+    def ensure_collection(self, vector_size: int) -> None:
+        collections = self.client.get_collections().collections
+        exists = any(collection.name == self.collection_name for collection in collections)
+        if not exists:
+            self.client.create_collection(
+                collection_name=self.collection_name,
+                vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE),
+            )
+
+    def upsert_chunks(self, chunks: list[Chunk], embeddings: list[list[float]]) -> None:
+        if len(chunks) != len(embeddings):
+            raise ValueError("Chunks and embeddings must have the same length.")
+        if not chunks:
+            return
+
+        self.ensure_collection(vector_size=len(embeddings[0]))
+        points = [
+            PointStruct(
+                id=chunk.id,
+                vector=embedding,
+                payload={
+                    "text": chunk.text,
+                    "chunk_index": chunk.index,
+                    "source_type": chunk.source_type.value,
+                    "source": chunk.source,
+                    "title": chunk.title,
+                    "metadata": chunk.metadata,
+                },
+            )
+            for chunk, embedding in zip(chunks, embeddings, strict=True)
+        ]
+        self.client.upsert(collection_name=self.collection_name, points=points)
+
+    def search(self, query_embedding: list[float], limit: int = 5) -> list[SearchResult]:
+        if hasattr(self.client, "query_points"):
+            response = self.client.query_points(
+                collection_name=self.collection_name,
+                query=query_embedding,
+                limit=limit,
+                with_payload=True,
+            )
+            hits = response.points
+        else:
+            hits = self.client.search(
+                collection_name=self.collection_name,
+                query_vector=query_embedding,
+                limit=limit,
+                with_payload=True,
+            )
+
+        results: list[SearchResult] = []
+        for hit in hits:
+            payload = hit.payload or {}
+            results.append(
+                SearchResult(
+                    score=float(hit.score),
+                    text=str(payload.get("text", "")),
+                    title=str(payload.get("title", "")),
+                    source=str(payload.get("source", "")),
+                    source_type=str(payload.get("source_type", "")),
+                    metadata=dict(payload.get("metadata", {})),
+                )
+            )
+        return results

app/ui/__init__.py

@@ -0,0 +1 @@
+"""UI package."""

app/ui/gradio_app.py

@@ -0,0 +1,341 @@
+import logging
+import traceback
+
+import gradio as gr
+
+from app.core.config import settings
+from app.ui.theme import CSS, HEAD, JS
+from app.utils.zerogpu import gpu
+
+logger = logging.getLogger(__name__)
+
+THEME = gr.themes.Base(
+    primary_hue="cyan",
+    secondary_hue="lime",
+    neutral_hue="slate",
+    radius_size="sm",
+    font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
+    font_mono=[gr.themes.GoogleFont("JetBrains Mono"), "ui-monospace", "monospace"],
+)
+
+
+def _format_metadata(metadata: dict) -> str:
+    if not metadata:
+        return "No metadata found."
+    rows = []
+    for key, value in metadata.items():
+        rows.append(f"**{key}**: {value}")
+    return "\n\n".join(rows)
+
+
+@gpu()
+def _ingest(url: str, pdf_file: str | None, chunk_size: int, chunk_overlap: int, collection_name: str):
+    logger.info(
+        "Ingest requested url=%s pdf_file=%s chunk_size=%s chunk_overlap=%s collection=%s",
+        url,
+        pdf_file,
+        chunk_size,
+        chunk_overlap,
+        collection_name,
+    )
+    try:
+        from app.services.ingestion import ingest_source
+
+        result = ingest_source(
+            url=url,
+            pdf_path=pdf_file,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            collection_name=collection_name,
+        )
+        document = result.document
+        status = (
+            f"### Ingestion complete\n\n"
+            f"Uploaded **{len(result.chunks)} chunks** into Qdrant collection "
+            f"`{result.collection_name}`.\n\n"
+            f"Saved extracted text to `{result.export_path}`."
+        )
+        preview = document.text[:12000]
+        if len(document.text) > len(preview):
+            preview += "\n\n[Preview truncated in UI. Full text is saved in the export file.]"
+        return (
+            status,
+            document.title,
+            document.source_type.value,
+            str(len(document.text)),
+            str(len(result.chunks)),
+            _format_metadata(document.metadata),
+            preview,
+            str(result.export_path),
+        )
+    except Exception as exc:
+        return (
+            f"### Ingestion failed\n\n`{type(exc).__name__}: {exc}`\n\n```text\n{traceback.format_exc(limit=2)}\n```",
+            "",
+            "",
+            "0",
+            "0",
+            "",
+            "",
+            "",
+        )
+
+
+@gpu()
+def _search(query: str, limit: int, collection_name: str):
+    logger.info("Search requested query=%s limit=%s collection=%s", query, limit, collection_name)
+    try:
+        from app.services.ingestion import search_knowledge_base
+
+        results = search_knowledge_base(query, limit=limit, collection_name=collection_name)
+    except Exception as exc:
+        if "MPS backend out of memory" in str(exc):
+            return (
+                "### Search failed\n\n"
+                "The local embedding model ran out of Apple GPU memory. "
+                "Restart the app so the new CPU embedding setting takes effect. "
+                "Keep `EMBEDDING_DEVICE=cpu` in `.env`."
+            )
+        return f"### Search failed\n\n`{type(exc).__name__}: {exc}`"
+
+    if not results:
+        return "No matches found."
+
+    blocks = []
+    for index, result in enumerate(results, start=1):
+        excerpt = result.text[:1200]
+        blocks.append(
+            "\n".join(
+                [
+                    f"### {index}. {result.title}",
+                    f"**Score:** {result.score:.4f}",
+                    f"**Source:** {result.source_type} | {result.source}",
+                    "",
+                    excerpt,
+                ]
+            )
+        )
+    return "\n\n---\n\n".join(blocks)
+
+
+@gpu()
+def _answer(query: str, limit: int, collection_name: str):
+    logger.info("Answer requested query=%s limit=%s collection=%s", query, limit, collection_name)
+    try:
+        from app.services.ingestion import answer_from_knowledge_base
+
+        result = answer_from_knowledge_base(query, limit=limit, collection_name=collection_name)
+    except Exception as exc:
+        if "MPS backend out of memory" in str(exc):
+            return (
+                "### Answer failed\n\n"
+                "The local embedding model ran out of Apple GPU memory. "
+                "Restart the app so the new CPU embedding setting takes effect. "
+                "Keep `EMBEDDING_DEVICE=cpu` in `.env`.",
+                "",
+                "",
+            )
+        return f"### Answer failed\n\n`{type(exc).__name__}: {exc}`", "", ""
+
+    context_blocks = []
+    for index, item in enumerate(result.context, start=1):
+        context_blocks.append(
+            "\n".join(
+                [
+                    f"### [{index}] {item.title}",
+                    f"**Score:** {item.score:.4f}",
+                    f"**Source:** {item.source_type} | {item.source}",
+                    "",
+                    item.text[:1000],
+                ]
+            )
+        )
+
+    reasoning = result.reasoning or "No reasoning content was returned by the API."
+    return result.answer, reasoning, "\n\n---\n\n".join(context_blocks)
+
+
+def build_app() -> gr.Blocks:
+    with gr.Blocks(
+        title=f"{settings.PROJECT_NAME} Ingestor",
+    ) as demo:
+        with gr.Column(elem_id="kh-shell"):
+            gr.Markdown(
+                f"""
+# {settings.PROJECT_NAME}
+Turn papers, PDFs, and videos into a searchable vector memory.
+
+Extract text, chunk it cleanly, embed locally, and use NVIDIA chat for grounded answers.
+""",
+                elem_id="kh-title",
+            )
+            gr.HTML(
+                f"""
+<div class="kh-chip-row">
+  <div class="kh-chip">Embeddings <code>{settings.NEMOTRON_EMBED_MODEL}</code></div>
+  <div class="kh-chip">Parser <code>{settings.NEMOTRON_PARSE_MODEL}</code></div>
+  <div class="kh-chip">Chat <code>{settings.NVIDIA_CHAT_MODEL}</code></div>
+  <div class="kh-chip">Collection <code>{settings.QDRANT_COLLECTION_NAME}</code></div>
+  <div class="kh-chip">Sources PDF · arXiv · YouTube</div>
+</div>
+""",
+            )
+
+            with gr.Tabs():
+                with gr.Tab("Ingest"):
+                    with gr.Row(equal_height=True):
+                        with gr.Column(scale=5, elem_classes=["kh-panel"]):
+                            gr.Markdown(
+                                "### Source Intake\n<div class='kh-subhead'>Upload a PDF or paste one link. The pipeline handles extraction, chunking, local embeddings, and Qdrant upload.</div>"
+                            )
+                            source_url = gr.Textbox(
+                                label="YouTube or arXiv input",
+                                placeholder="Paste a YouTube URL, arXiv URL, or arXiv ID",
+                                lines=2,
+                            )
+                            pdf_file = gr.File(
+                                label="PDF document",
+                                file_types=[".pdf"],
+                                type="filepath",
+                            )
+                            with gr.Row():
+                                chunk_size = gr.Slider(
+                                    400,
+                                    2500,
+                                    value=settings.CHUNK_SIZE,
+                                    step=50,
+                                    label="Chunk size",
+                                )
+                                chunk_overlap = gr.Slider(
+                                    0,
+                                    600,
+                                    value=settings.CHUNK_OVERLAP,
+                                    step=25,
+                                    label="Chunk overlap",
+                                )
+                            collection_name_ingest = gr.Textbox(
+                                label="Collection Name",
+                                value=settings.QDRANT_COLLECTION_NAME,
+                                placeholder="Enter Qdrant collection name",
+                            )
+                            ingest_btn = gr.Button("Ingest into Qdrant", variant="primary")
+
+                        with gr.Column(scale=4, elem_classes=["kh-panel"]):
+                            gr.Markdown("### Pipeline Status")
+                            status = gr.Markdown(elem_id="kh-status")
+                            with gr.Row():
+                                title = gr.Textbox(
+                                    label="Title",
+                                    interactive=False,
+                                    elem_classes=["kh-stat"],
+                                )
+                                source_type = gr.Textbox(
+                                    label="Type",
+                                    interactive=False,
+                                    elem_classes=["kh-stat"],
+                                )
+                            with gr.Row():
+                                char_count = gr.Textbox(
+                                    label="Characters",
+                                    interactive=False,
+                                    elem_classes=["kh-stat"],
+                                )
+                                chunk_count = gr.Textbox(
+                                    label="Chunks",
+                                    interactive=False,
+                                    elem_classes=["kh-stat"],
+                                )
+                            export_path = gr.Textbox(label="Export file", interactive=False)
+
+                    with gr.Row(equal_height=True):
+                        metadata = gr.Markdown(label="Metadata", elem_classes=["kh-panel"])
+                        text_preview = gr.Textbox(
+                            label="Extracted text preview",
+                            lines=18,
+                            interactive=False,
+                            elem_id="kh-text-preview",
+                            elem_classes=["kh-panel"],
+                        )
+
+                    ingest_btn.click(
+                        fn=_ingest,
+                        inputs=[source_url, pdf_file, chunk_size, chunk_overlap, collection_name_ingest],
+                        outputs=[
+                            status,
+                            title,
+                            source_type,
+                            char_count,
+                            chunk_count,
+                            metadata,
+                            text_preview,
+                            export_path,
+                        ],
+                    )
+
+                with gr.Tab("Retrieve"):
+                    with gr.Row(equal_height=True):
+                        with gr.Column(scale=3, elem_classes=["kh-panel"]):
+                            gr.Markdown(
+                                "### Retrieval Probe\n<div class='kh-subhead'>Run a quick similarity search against the same Qdrant collection after ingestion.</div>"
+                            )
+                            query = gr.Textbox(
+                                label="Search query",
+                                placeholder="Ask a question or enter keywords",
+                                lines=4,
+                            )
+                            limit = gr.Slider(1, 10, value=5, step=1, label="Results")
+                            collection_name_retrieve = gr.Textbox(
+                                label="Collection Name",
+                                value=settings.QDRANT_COLLECTION_NAME,
+                                placeholder="Enter Qdrant collection name",
+                            )
+                            with gr.Row():
+                                search_btn = gr.Button("Search Qdrant", variant="secondary")
+                                answer_btn = gr.Button("Answer with NVIDIA", variant="primary")
+                        with gr.Column(scale=5, elem_classes=["kh-panel"]):
+                            gr.Markdown("### Answer")
+                            answer_output = gr.Markdown(elem_id="kh-answer")
+
+                    with gr.Row(equal_height=True):
+                        with gr.Column(elem_classes=["kh-panel"]):
+                            gr.Markdown("### Matches")
+                            search_results = gr.Markdown(elem_id="kh-search-results")
+                        with gr.Column(elem_classes=["kh-panel"]):
+                            gr.Markdown("### Reasoning")
+                            reasoning_output = gr.Markdown(elem_id="kh-reasoning")
+
+                    search_btn.click(
+                        fn=_search,
+                        inputs=[query, limit, collection_name_retrieve],
+                        outputs=search_results,
+                    )
+                    answer_btn.click(
+                        fn=_answer,
+                        inputs=[query, limit, collection_name_retrieve],
+                        outputs=[answer_output, reasoning_output, search_results],
+                    )
+
+            # Sync collection names across tabs
+            collection_name_ingest.change(
+                fn=lambda x: x, inputs=[collection_name_ingest], outputs=[collection_name_retrieve]
+            )
+            collection_name_retrieve.change(
+                fn=lambda x: x, inputs=[collection_name_retrieve], outputs=[collection_name_ingest]
+            )
+
+    return demo
+
+
+def serve() -> None:
+    logger.info("Building Gradio app")
+    demo = build_app()
+    logger.info("Launching Gradio server on 0.0.0.0:7860")
+    demo.queue().launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        show_error=True,
+        theme=THEME,
+        css=CSS,
+        js=JS,
+        head=HEAD,
+    )

app/ui/theme.py

@@ -0,0 +1,258 @@
+HEAD = """
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;600&display=swap" rel="stylesheet">
+"""
+
+JS = """
+() => {
+  const root = document.querySelector('.gradio-container');
+  if (!root) return;
+  root.dataset.ready = 'true';
+  const marker = document.createElement('div');
+  marker.className = 'kh-scanline';
+  root.prepend(marker);
+}
+"""
+
+CSS = """
+:root {
+  --kh-bg: #080b0f;
+  --kh-surface: rgba(18, 24, 32, 0.78);
+  --kh-surface-strong: rgba(27, 36, 48, 0.92);
+  --kh-ink: #f7fbff;
+  --kh-muted: #a7b4c2;
+  --kh-soft: #d8e1ea;
+  --kh-line: rgba(255, 255, 255, 0.12);
+  --kh-cyan: #20d6c7;
+  --kh-lime: #b8f45d;
+  --kh-rose: #ff6b8a;
+  --kh-amber: #ffcf5c;
+  --kh-shadow: rgba(0, 0, 0, 0.32);
+}
+
+.gradio-container {
+  min-height: 100vh;
+  background:
+    radial-gradient(circle at 18% 8%, rgba(32, 214, 199, 0.22), transparent 30%),
+    radial-gradient(circle at 86% 12%, rgba(255, 207, 92, 0.16), transparent 28%),
+    linear-gradient(135deg, #080b0f 0%, #101720 48%, #0b1017 100%) !important;
+  color: var(--kh-ink);
+  font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+}
+
+.kh-scanline {
+  position: fixed;
+  inset: 0;
+  pointer-events: none;
+  background-image: linear-gradient(rgba(255,255,255,0.035) 1px, transparent 1px);
+  background-size: 100% 4px;
+  mask-image: linear-gradient(to bottom, transparent, black 18%, black 72%, transparent);
+  opacity: 0.18;
+  z-index: 0;
+}
+
+#kh-shell {
+  position: relative;
+  z-index: 1;
+  max-width: 1220px;
+  margin: 0 auto;
+  padding: 28px 18px 42px;
+}
+
+#kh-title {
+  padding: 34px 0 22px;
+  border-bottom: 1px solid var(--kh-line);
+}
+
+#kh-title h1 {
+  max-width: 920px;
+  color: var(--kh-ink);
+  font-size: clamp(2.6rem, 6vw, 6rem);
+  font-weight: 800;
+  line-height: 0.9;
+  margin: 0 0 14px;
+  letter-spacing: 0;
+}
+
+#kh-title p {
+  max-width: 780px;
+  color: var(--kh-muted);
+  font-size: 1.04rem;
+  line-height: 1.65;
+}
+
+#kh-title code,
+.kh-chip code {
+  color: var(--kh-lime);
+  background: rgba(184, 244, 93, 0.09);
+  border: 1px solid rgba(184, 244, 93, 0.18);
+  border-radius: 6px;
+  padding: 2px 6px;
+  font-family: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
+}
+
+.kh-panel {
+  border: 1px solid var(--kh-line);
+  border-radius: 8px;
+  background: linear-gradient(180deg, var(--kh-surface-strong), var(--kh-surface));
+  box-shadow: 0 24px 70px var(--kh-shadow);
+  backdrop-filter: blur(18px);
+  padding: 18px !important;
+}
+
+.kh-panel label,
+.kh-panel .label-wrap span {
+  color: var(--kh-soft) !important;
+  font-weight: 700 !important;
+}
+
+.kh-subhead {
+  margin: 8px 0 16px;
+  color: var(--kh-muted);
+  font-size: 0.95rem;
+}
+
+.kh-chip-row {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 10px;
+  margin-top: 18px;
+}
+
+.kh-chip {
+  border: 1px solid var(--kh-line);
+  border-radius: 999px;
+  padding: 8px 12px;
+  color: var(--kh-soft);
+  background: rgba(255, 255, 255, 0.055);
+  font-size: 0.9rem;
+}
+
+.kh-stat {
+  min-height: 92px;
+  border: 1px solid var(--kh-line);
+  border-radius: 8px;
+  padding: 14px 16px;
+  background: rgba(255, 255, 255, 0.055);
+}
+
+.kh-stat .wrap,
+.kh-stat input {
+  background: transparent !important;
+}
+
+.tabs {
+  margin-top: 20px;
+}
+
+.tab-nav button {
+  color: var(--kh-muted) !important;
+  border-radius: 8px !important;
+  font-weight: 700 !important;
+}
+
+.tab-nav button.selected {
+  color: var(--kh-ink) !important;
+  background: linear-gradient(135deg, rgba(32, 214, 199, 0.22), rgba(184, 244, 93, 0.12)) !important;
+  border: 1px solid rgba(32, 214, 199, 0.34) !important;
+}
+
+textarea,
+input {
+  color: var(--kh-ink) !important;
+  background: rgba(3, 7, 12, 0.52) !important;
+  border-color: rgba(255, 255, 255, 0.12) !important;
+  font-size: 0.96rem !important;
+}
+
+textarea::placeholder,
+input::placeholder {
+  color: rgba(216, 225, 234, 0.46) !important;
+}
+
+#kh-status {
+  min-height: 130px;
+}
+
+#kh-status h3 {
+  color: var(--kh-lime);
+  margin-top: 0;
+}
+
+#kh-text-preview textarea {
+  min-height: 430px !important;
+  line-height: 1.6 !important;
+  font-family: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, monospace !important;
+  font-size: 0.9rem !important;
+}
+
+#kh-search-results {
+  min-height: 410px;
+}
+
+#kh-answer,
+#kh-reasoning {
+  min-height: 240px;
+}
+
+#kh-answer {
+  font-size: 1.02rem;
+  line-height: 1.7;
+}
+
+#kh-search-results h3 {
+  color: var(--kh-cyan);
+}
+
+#kh-reasoning {
+  color: var(--kh-muted);
+  font-family: "JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
+  font-size: 0.86rem;
+  line-height: 1.6;
+}
+
+.prose,
+.markdown {
+  color: var(--kh-soft) !important;
+}
+
+.prose strong,
+.markdown strong {
+  color: var(--kh-ink) !important;
+}
+
+button.primary {
+  min-height: 46px;
+  background: linear-gradient(135deg, var(--kh-cyan), var(--kh-lime)) !important;
+  color: #061015 !important;
+  border: 0 !important;
+  border-radius: 8px !important;
+  font-weight: 800 !important;
+  box-shadow: 0 16px 34px rgba(32, 214, 199, 0.2);
+}
+
+button.secondary {
+  border-radius: 8px !important;
+}
+
+.file-preview,
+.upload-container {
+  border-color: rgba(32, 214, 199, 0.26) !important;
+  background: rgba(32, 214, 199, 0.055) !important;
+}
+
+@media (max-width: 760px) {
+  #kh-shell {
+    padding: 18px 10px 32px;
+  }
+
+  #kh-title h1 {
+    font-size: clamp(2.25rem, 15vw, 4.2rem);
+  }
+
+  .kh-panel {
+    padding: 14px !important;
+  }
+}
+"""

app/utils/__init__.py

@@ -0,0 +1 @@
+"""Utility helpers."""

app/utils/source_detection.py

@@ -0,0 +1,37 @@
+import re
+from pathlib import Path
+from urllib.parse import urlparse
+
+from app.core.models import SourceType
+
+
+ARXIV_RE = re.compile(r"(?:arxiv\.org/(?:abs|pdf)/)?(?P<id>\d{4}\.\d{4,5})(?:v\d+)?", re.I)
+YOUTUBE_HOSTS = {"youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be", "www.youtu.be"}
+
+
+def detect_source(url: str | None, pdf_path: str | None) -> SourceType:
+    if pdf_path:
+        suffix = Path(pdf_path).suffix.lower()
+        if suffix == ".pdf":
+            return SourceType.PDF
+        raise ValueError("Uploaded file must be a PDF.")
+
+    if not url or not url.strip():
+        raise ValueError("Provide a YouTube link, arXiv link/ID, or upload a PDF.")
+
+    clean_url = url.strip()
+    parsed = urlparse(clean_url)
+    host = parsed.netloc.lower()
+
+    if host in YOUTUBE_HOSTS:
+        return SourceType.YOUTUBE
+    if "arxiv.org" in host or ARXIV_RE.search(clean_url):
+        return SourceType.ARXIV
+    raise ValueError("Could not detect source type. Use a YouTube URL, arXiv URL/ID, or PDF.")
+
+
+def extract_arxiv_id(value: str) -> str:
+    match = ARXIV_RE.search(value.strip())
+    if not match:
+        raise ValueError("Could not find a valid arXiv ID.")
+    return match.group("id")

app/utils/zerogpu.py

@@ -0,0 +1,36 @@
+import os
+from collections.abc import Callable
+from typing import TypeVar
+
+from app.core.config import settings
+
+
+F = TypeVar("F", bound=Callable)
+
+
+def gpu(duration: int | None = None) -> Callable[[F], F]:
+    if not _should_use_zerogpu():
+        return _identity
+
+    try:
+        import spaces
+    except ImportError:
+        return _identity
+
+    return spaces.GPU(duration=duration or settings.ZEROGPU_DURATION_SECONDS)
+
+
+def _identity(func: F) -> F:
+    return func
+
+
+def _should_use_zerogpu() -> bool:
+    if os.getenv("DISABLE_ZEROGPU", "").lower() in {"1", "true", "yes"}:
+        return False
+    if os.getenv("ENABLE_ZEROGPU", "").lower() in {"1", "true", "yes"}:
+        return True
+    return bool(os.getenv("SPACE_ID"))
+
+
+def is_enabled() -> bool:
+    return _should_use_zerogpu()

pyproject.toml

@@ -0,0 +1,31 @@
+[project]
+name = "knowledgehub-ingestor"
+version = "1.0.0"
+description = "A Gradio document ingestion UI for PDFs, arXiv papers, and YouTube transcripts."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "arxiv>=2.1.3",
+    "datasets>=5.0.0",
+    "gradio>=5.0.0",
+    "openai>=1.99.0",
+    "pydantic-settings>=2.4.0",
+    "pypdf>=4.3.1",
+    "python-dotenv>=1.0.1",
+    "qdrant-client>=1.12.1",
+    "requests>=2.32.3",
+    "sentence-transformers>=3.0.1",
+    "spaces",
+    "torchvision>=0.27.0",
+    "youtube-transcript-api>=0.6.2",
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff>=0.6.0",
+    "pytest>=8.3.2",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"

requirements.txt

@@ -0,0 +1,11 @@
+arxiv>=2.1.3
+gradio>=5.0.0
+openai>=1.99.0
+pydantic-settings>=2.4.0
+pypdf>=4.3.1
+python-dotenv>=1.0.1
+qdrant-client>=1.12.1
+requests>=2.32.3
+sentence-transformers>=3.0.1
+spaces
+youtube-transcript-api>=0.6.2