import joblib import os import json import time from typing import Any, Dict, Optional from sklearn.pipeline import Pipeline from src.monitoring.logger import get_logger logger = get_logger(__name__) class ModelArtifactTracker: """Manages serialization of the unified ML pipeline, SHAP explainer, and feature metadata.""" ARTIFACT_DIR = "models" RUNS_DIR = "models/runs" PIPELINE_FILE = "unified_pipeline.joblib" EXPLAINER_FILE = "shap_explainer.joblib" METADATA_FILE = "model_metadata.json" @classmethod def initialize(cls): os.makedirs(cls.ARTIFACT_DIR, exist_ok=True) os.makedirs(cls.RUNS_DIR, exist_ok=True) @classmethod def save_pipeline(cls, pipeline: Pipeline, explainer: Any = None, metadata: Optional[Dict] = None) -> str: """ Saves the pipeline, optional SHAP explainer, and optional feature metadata. metadata format: { "features": { col: {"type": "number"|"text", "values": [...] or null} }, "task_type": "classification" | "regression", "target_col": str, "model_name": str, } """ cls.initialize() file_path = os.path.join(cls.ARTIFACT_DIR, cls.PIPELINE_FILE) expl_path = os.path.join(cls.ARTIFACT_DIR, cls.EXPLAINER_FILE) meta_path = os.path.join(cls.ARTIFACT_DIR, cls.METADATA_FILE) try: joblib.dump(pipeline, file_path) if explainer is not None: joblib.dump(explainer, expl_path) logger.info(f"SHAP Explainer serialized at {expl_path}.") if metadata is not None: with open(meta_path, "w") as f: json.dump(metadata, f, indent=2) logger.info(f"Feature metadata saved at {meta_path}.") size_mb = os.path.getsize(file_path) / (1024 * 1024) logger.info(f"Unified Pipeline serialized at {file_path}. Size: {size_mb:.2f} MB") return file_path except Exception as e: logger.error(f"Failed to serialize pipeline or explainer: {str(e)}") raise e @classmethod def load_artifacts(cls): """Loads the unified pipeline and optional SHAP explainer for inference.""" file_path = os.path.join(cls.ARTIFACT_DIR, cls.PIPELINE_FILE) expl_path = os.path.join(cls.ARTIFACT_DIR, cls.EXPLAINER_FILE) if not os.path.exists(file_path): raise FileNotFoundError("Missing unified pipeline. Train a model first.") logger.info(f"Loading unified pipeline from {file_path}") pipeline = joblib.load(file_path) explainer = None if os.path.exists(expl_path): logger.info(f"Loading SHAP explainer from {expl_path}") explainer = joblib.load(expl_path) return pipeline, explainer @classmethod def load_metadata(cls) -> Optional[Dict]: """Loads the saved feature metadata, or returns None if not present.""" meta_path = os.path.join(cls.ARTIFACT_DIR, cls.METADATA_FILE) if not os.path.exists(meta_path): return None with open(meta_path, "r") as f: return json.load(f) @classmethod def log_experiment(cls, metrics: dict, optuna_params: dict, llm_overrides: dict) -> str: cls.initialize() run_id = f"run_{int(time.time())}" run_file = os.path.join(cls.RUNS_DIR, f"{run_id}.json") snapshot = { "run_id": run_id, "metrics": metrics, "best_hyperparameters": optuna_params, "llm_overrides": llm_overrides, } with open(run_file, "w") as f: json.dump(snapshot, f, indent=4) logger.info(f"Experiment versioned at {run_file}") return run_id