SAP-ERP-AI-Agent / src /config.py
daisysooyeon's picture
perf: reranker -> bge-reranker-base (hit 88.3% in multi-query, 3x faster than v2-m3; MiniLM too lossy)
ba75f43
Raw
History Blame Contribute Delete
10.1 kB
"""
src/config.py
Central configuration loader for SAP ERP AI Agent.
Reads configs.yaml from the project root and exposes a typed AppConfig object.
Sensitive secrets (API keys, etc.) remain in .env and are accessed via os.getenv.
Usage:
from src.config import get_config
cfg = get_config()
print(cfg.models.router.name) # "qwen3:4b"
print(cfg.paths.erp_db) # "data/sap_erp.db"
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
import yaml
# ---------------------------------------------------------------------------
# Typed schema (dataclasses)
# ---------------------------------------------------------------------------
@dataclass
class ModelConfig:
name: str
temperature: float = 0.0
provider: str = "ollama" # "ollama" | "openrouter"
@dataclass
class OpenRouterConfig:
base_url: str = "https://openrouter.ai/api/v1"
@dataclass
class ModelsConfig:
preprocessor: ModelConfig = field(default_factory=lambda: ModelConfig("openai/gpt-4o-mini", 0.0, provider="openrouter"))
router: ModelConfig = field(default_factory=lambda: ModelConfig("qwen3:4b"))
worker_a: ModelConfig = field(default_factory=lambda: ModelConfig("qwen/qwen3-8b", provider="openrouter"))
worker_a_sql: ModelConfig = field(default_factory=lambda: ModelConfig("qwen/qwen3-coder-30b-a3b-instruct", provider="openrouter"))
worker_b: ModelConfig = field(default_factory=lambda: ModelConfig("qwen3:4b", 0.1))
synthesizer: ModelConfig = field(default_factory=lambda: ModelConfig("gpt-4o", 0.3))
data_gen: ModelConfig = field(default_factory=lambda: ModelConfig("qwen/qwen3-8b", 0.5, provider="openrouter"))
eval_judge: ModelConfig = field(default_factory=lambda: ModelConfig("openai/gpt-4o-mini", 0.0, provider="openrouter"))
@dataclass
class OllamaConfig:
base_url: str = "http://localhost:11434"
@dataclass
class PathsConfig:
erp_db: str = "data/sap_erp.db"
checkpoint_db: str = "data/checkpoints.db"
chroma_db: str = "./chroma_db"
docs_dir: str = "data/docs"
eval_test_cases: str = "data/eval/router_test_cases.json"
reports_dir: str = "reports"
@dataclass
class LoggingConfig:
level: str = "DEBUG"
console_level: str = "INFO"
dir: str = "logs"
max_bytes: int = 10 * 1024 * 1024 # 10 MB
backup_count: int = 5
@dataclass
class RagConfig:
chunk_size: int = 512
chunk_overlap: int = 64
collection_name: str = "sap_manuals"
dense_weight: float = 0.6
sparse_weight: float = 0.4
top_k_retrieval: int = 10
top_k_rerank: int = 3
contextual_header: bool = True # 청크 λ³Έλ¬Έ μ•žμ— [Source: ... | p.N] 헀더λ₯Ό prepend ν•΄ μž„λ² λ”©μ— 메타 반영
ocr_enabled: bool = True # PDF νŽ˜μ΄μ§€ λ‚΄ 이미지에 Surya OCR 적용 β†’ 별도 OCR 청크 생성
rerank_blend_alpha: float = 0.8 # 리랭크 점수 λΈ”λ Œλ”© κ°€μ€‘μΉ˜: final = alpha*cross_encoder + (1-alpha)*κ²€μƒ‰μˆœμœ„. 1.0이면 순수 리랭컀. (ν†΅μ œ μŠ€μœ• κ²°κ³Ό 0.8이 hit/ndcg/mrr λ™μ‹œ 졜적)
meta_boost_beta: float = 0.0 # 메타데이터 μ†Œν”„νŠΈ λΆ€μŠ€νŠΈ κ°€μ€‘μΉ˜: final += beta*(쿼리↔unit/lesson/section 토큰 일치). 0이면 λΉ„ν™œμ„±
reranker_model: str = "BAAI/bge-reranker-base" # cross-encoder 리랭컀 (멀티쿼리 벀치: hit 88.3% 졜적, v2-m3 λŒ€λΉ„ ~3x 빠름)
@dataclass
class FeatureFlagsConfig:
human_in_the_loop: bool = True
@dataclass
class AppConfig:
ollama: OllamaConfig = field(default_factory=OllamaConfig)
openrouter: OpenRouterConfig = field(default_factory=OpenRouterConfig)
models: ModelsConfig = field(default_factory=ModelsConfig)
paths: PathsConfig = field(default_factory=PathsConfig)
logging: LoggingConfig = field(default_factory=LoggingConfig)
rag: RagConfig = field(default_factory=RagConfig)
feature_flags: FeatureFlagsConfig = field(default_factory=FeatureFlagsConfig)
# ---------------------------------------------------------------------------
# Loader
# ---------------------------------------------------------------------------
_CONFIG_PATH = Path(__file__).resolve().parent.parent / "configs.yaml"
def _parse_model(raw: dict) -> ModelConfig:
return ModelConfig(
name=raw.get("name", "qwen3:4b"),
temperature=float(raw.get("temperature", 0.0)),
provider=raw.get("provider", "ollama"),
)
def _load_from_yaml(path: Path) -> AppConfig:
"""Parse configs.yaml and build a fully typed AppConfig."""
with open(path, encoding="utf-8") as f:
raw: dict = yaml.safe_load(f) or {}
# ── ollama ───────────────────────────────────────────────────────
o = raw.get("ollama", {})
ollama = OllamaConfig(
base_url=o.get("base_url", "http://localhost:11434"),
)
# ── openrouter ────────────────────────────────────────────────
or_ = raw.get("openrouter", {})
openrouter = OpenRouterConfig(
base_url=or_.get("base_url", "https://openrouter.ai/api/v1"),
)
# ── models ──────────────────────────────────────────────────────────────
m = raw.get("models", {})
models = ModelsConfig(
router= _parse_model(m.get("router", {"name": "qwen3:4b"})),
worker_a= _parse_model(m.get("worker_a", {"name": "qwen/qwen3-8b", "provider": "openrouter"})),
worker_a_sql=_parse_model(m.get("worker_a_sql",{"name": "qwen/qwen3-coder-30b-a3b-instruct","provider": "openrouter"})),
worker_b= _parse_model(m.get("worker_b", {"name": "qwen3:4b", "temperature": 0.1})),
synthesizer= _parse_model(m.get("synthesizer", {"name": "gpt-4o", "temperature": 0.3})),
data_gen= _parse_model(m.get("data_gen", {"name": "qwen/qwen3-8b", "temperature": 0.5, "provider": "openrouter"})),
eval_judge= _parse_model(m.get("eval_judge", {"name": "openai/gpt-4o-mini", "temperature": 0.0, "provider": "openrouter"})),
)
# ── paths ────────────────────────────────────────────────────────────────
p = raw.get("paths", {})
paths = PathsConfig(
erp_db= p.get("erp_db", "data/sap_erp.db"),
checkpoint_db= p.get("checkpoint_db", "data/checkpoints.db"),
chroma_db= p.get("chroma_db", "./chroma_db"),
docs_dir= p.get("docs_dir", "data/docs"),
eval_test_cases= p.get("eval_test_cases", "data/eval/router_test_cases.json"),
reports_dir= p.get("reports_dir", "reports"),
)
# ── logging ─────────────────────────────────────────────────────────────
lg = raw.get("logging", {})
logging = LoggingConfig(
level= lg.get("level", "DEBUG"),
console_level= lg.get("console_level", "INFO"),
dir= lg.get("dir", "logs"),
max_bytes= int(lg.get("max_bytes", 10 * 1024 * 1024)),
backup_count= int(lg.get("backup_count", 5)),
)
# ── rag ─────────────────────────────────────────────────────────────────
r = raw.get("rag", {})
rag = RagConfig(
chunk_size= int(r.get("chunk_size", 512)),
chunk_overlap= int(r.get("chunk_overlap", 64)),
collection_name= r.get("collection_name", "sap_manuals"),
dense_weight= float(r.get("dense_weight", 0.6)),
sparse_weight= float(r.get("sparse_weight", 0.4)),
top_k_retrieval= int(r.get("top_k_retrieval", 10)),
top_k_rerank= int(r.get("top_k_rerank", 3)),
contextual_header= bool(r.get("contextual_header", True)),
ocr_enabled= bool(r.get("ocr_enabled", True)),
rerank_blend_alpha=float(r.get("rerank_blend_alpha", 0.8)),
meta_boost_beta=float(r.get("meta_boost_beta", 0.0)),
reranker_model= r.get("reranker_model", "BAAI/bge-reranker-base"),
)
# ── feature flags ────────────────────────────────────────────────────────
ff = raw.get("feature_flags", {})
feature_flags = FeatureFlagsConfig(
human_in_the_loop=bool(ff.get("human_in_the_loop", True)),
)
return AppConfig(
ollama=ollama,
openrouter=openrouter,
models=models,
paths=paths,
logging=logging,
rag=rag,
feature_flags=feature_flags,
)
@lru_cache(maxsize=1)
def get_config() -> AppConfig:
"""
Load and return the AppConfig singleton (cached after first call).
The config path can be overridden via the CONFIG_PATH env variable:
CONFIG_PATH=/custom/path/configs.yaml python -m src.main
"""
config_path = Path(os.getenv("CONFIG_PATH", str(_CONFIG_PATH)))
if not config_path.exists():
raise FileNotFoundError(
f"configs.yaml not found at {config_path}. "
"Make sure configs.yaml exists in the project root."
)
return _load_from_yaml(config_path)