from dataclasses import dataclass, field from pathlib import Path @dataclass class Config: # Crawl settings default_url: str = "https://example.com" default_depth: int = 2 default_max_pages: int = 50 max_depth: int = 5 max_pages_limit: int = 500 # Chunking settings chunk_max_chars: int = 800 chunk_overlap: int = 100 # Model settings embedding_model: str = "all-MiniLM-L6-v2" embedding_model_fallback: str = "BAAI/bge-small-en-v1.5" reranker_model: str = "cross-encoder/ms-marco-MiniLM-L-6-v2" embedding_dim: int = 384 # Local LLM settings llm_model: str = "Qwen/Qwen2.5-0.5B-Instruct" llm_max_tokens: int = 1024 llm_temperature: float = 0.1 # Retrieval settings top_k_dense: int = 20 top_k_bm25: int = 20 top_k_rerank: int = 10 rrf_k: int = 60 # Storage settings data_dir: Path = field(default_factory=lambda: Path("data")) dataset_repo_id: str = "chatbot-data" # Paths pages_path: str = "pages.parquet" chunks_path: str = "chunks.parquet" facts_path: str = "facts.parquet" faiss_index_path: str = "faiss.index" bm25_index_path: str = "bm25_index.pkl" metadata_path: str = "metadata.json" @property def pages_file(self) -> Path: return self.data_dir / self.pages_path @property def chunks_file(self) -> Path: return self.data_dir / self.chunks_path @property def facts_file(self) -> Path: return self.data_dir / self.facts_path @property def faiss_file(self) -> Path: return self.data_dir / self.faiss_index_path @property def bm25_file(self) -> Path: return self.data_dir / self.bm25_index_path @property def metadata_file(self) -> Path: return self.data_dir / self.metadata_path