Download backend/flagship/ml.py from Gaurav711/HyperFlow: direct link, hf CLI and curl.
- Browser
- Download file 28.3 kB
-
https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/backend/flagship/ml.py
- Command line
-
hf download hf://spaces/Gaurav711/HyperFlow/backend/flagship/ml.py
-
curl -L -o ml.py https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/backend/flagship/ml.py
28.3 kB
| """Reproducible offline studies and an immutable, local model registry. | |
| No imports from the application, database, credentials, or provider clients. A | |
| public-data study is not evidence of quick-commerce demand or hidden sales. | |
| """ | |
| from __future__ import annotations | |
| from contextlib import contextmanager | |
| from datetime import datetime, timezone | |
| import hashlib | |
| import io | |
| import importlib.metadata | |
| import platform | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import pickle | |
| import tempfile | |
| import threading | |
| import zipfile | |
| from typing import Any | |
| import numpy as np | |
| import pandas as pd | |
| FEATURES = ("lag_1", "lag_7", "mean_7", "mean_28", "weekday") | |
| UCI_URL = "https://archive.ics.uci.edu/static/public/502/online+retail+ii.zip" | |
| UCI_CITATION = "Chen, D. (2012). Online Retail II. https://doi.org/10.24432/C5CG6D (CC BY 4.0)" | |
| # Pinned from the official HTTPS download used for the reproducible study. | |
| # A caller-written sidecar alone cannot establish public dataset identity. | |
| UCI_WORKBOOK_SHA256 = "bcbe73b35f5b7babf197fb0cb983a11f5d9ff929078d4aa53d171b1f2df2e980" | |
| UCI_ARCHIVE_SHA256 = "572e36277c2390fbfde10664750731e0a86f55e33470d91919085f0408e67bfb" | |
| _registry_lock = threading.RLock() | |
| _model_cache: dict[str, tuple[str, Any]] = {} | |
| def _json(value: Any) -> bytes: | |
| return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() | |
| def _hash(data: bytes) -> str: | |
| return hashlib.sha256(data).hexdigest() | |
| def _atomic(path: Path, data: bytes) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| descriptor, name = tempfile.mkstemp(prefix=".pending-", dir=path.parent) | |
| try: | |
| with os.fdopen(descriptor, "wb") as stream: | |
| stream.write(data) | |
| stream.flush() | |
| os.fsync(stream.fileno()) | |
| os.replace(name, path) | |
| finally: | |
| if os.path.exists(name): | |
| os.unlink(name) | |
| def _root(artifacts_dir: str | Path | None = None) -> Path: | |
| return Path(artifacts_dir or os.environ.get("HYPERFLOW_ARTIFACT_DIR", ".local/artifacts")).resolve() | |
| def _locked(root: Path): | |
| """Serialize registry pointer writes across threads and worker processes.""" | |
| import fcntl | |
| root.mkdir(parents=True, exist_ok=True) | |
| with _registry_lock, (root / ".registry.lock").open("a+b") as stream: | |
| fcntl.flock(stream, fcntl.LOCK_EX) | |
| try: | |
| yield | |
| finally: | |
| fcntl.flock(stream, fcntl.LOCK_UN) | |
| def metrics(actual, predicted) -> dict: | |
| actual, predicted = np.asarray(actual, float), np.asarray(predicted, float) | |
| if actual.size == 0 or actual.shape != predicted.shape: | |
| raise ValueError("Metric arrays must have equal, non-empty shapes") | |
| if not np.isfinite(actual).all() or not np.isfinite(predicted).all(): | |
| raise ValueError("Metrics require finite inputs") | |
| error = np.abs(actual - predicted) | |
| denominator = float(np.abs(actual).sum()) | |
| return {"mae": float(error.mean()), "rmse": float(np.sqrt(np.mean(error ** 2))), | |
| "wmape": float(error.sum() / denominator) if denominator else None, | |
| "observations": int(actual.size)} | |
| def read_retail(input_path: str | Path) -> tuple[pd.DataFrame, pd.DataFrame, dict]: | |
| """Read both original workbook sheets, or an explicitly supplied CSV. | |
| Cancellation invoices and negative quantities are separated before training. | |
| Missing customer IDs do not invalidate a product/day forecasting observation. | |
| Zero sales mean no recorded positive sales, never proven stockouts. | |
| """ | |
| path = Path(input_path) | |
| # Parse and fingerprint the same byte snapshot even if an input is replaced | |
| # while ingestion is running. | |
| input_bytes = path.read_bytes() | |
| if path.suffix.lower() == ".csv": | |
| frame = pd.read_csv(io.BytesIO(input_bytes)) | |
| elif path.suffix.lower() in {".xlsx", ".xls"}: | |
| try: | |
| frame = pd.concat(pd.read_excel(io.BytesIO(input_bytes), sheet_name=None).values(), ignore_index=True) | |
| except ImportError as exc: | |
| raise ValueError("Reading .xlsx requires openpyxl; CSV input remains available") from exc | |
| else: | |
| raise ValueError("Input must be .csv or .xlsx/.xls") | |
| aliases = {"invoice": "InvoiceNo", "invoiceno": "InvoiceNo", "stockcode": "StockCode", | |
| "quantity": "Quantity", "invoicedate": "InvoiceDate", "price": "UnitPrice", | |
| "unitprice": "UnitPrice", "country": "Country"} | |
| frame = frame.rename(columns={column: aliases.get(str(column).lower().replace(" ", ""), column) | |
| for column in frame.columns}) | |
| required = {"InvoiceNo", "StockCode", "Quantity", "InvoiceDate", "UnitPrice"} | |
| if not required.issubset(frame.columns): | |
| raise ValueError(f"Missing required retail columns: {sorted(required - set(frame.columns))}") | |
| frame["Quantity"] = pd.to_numeric(frame.Quantity, errors="coerce") | |
| frame["UnitPrice"] = pd.to_numeric(frame.UnitPrice, errors="coerce") | |
| frame["InvoiceDate"] = pd.to_datetime(frame.InvoiceDate, errors="coerce", format="mixed") | |
| valid = (frame.InvoiceDate.notna() & frame.StockCode.notna() & frame.InvoiceNo.notna() | |
| & np.isfinite(frame.Quantity) & np.isfinite(frame.UnitPrice) & (frame.UnitPrice >= 0)) | |
| clean = frame.loc[valid].copy() | |
| cancellation = clean.InvoiceNo.astype(str).str.upper().str.startswith("C") | (clean.Quantity < 0) | |
| returns = clean.loc[cancellation].copy() | |
| sales = clean.loc[~cancellation & (clean.Quantity > 0)].copy() | |
| sales["StockCode"] = sales.StockCode.astype(str) | |
| sales["day"] = sales.InvoiceDate.dt.normalize() | |
| # Unknown uploaded data does not inherit the original dataset's domain/currency. | |
| metadata = {"source_kind": "user_input", "source_format": "uci_online_retail_ii_schema", | |
| "dataset_sha256": _hash(input_bytes), "input_rows": len(frame), | |
| "positive_sales_rows": len(sales), "return_cancellation_rows": len(returns), | |
| "invalid_rows": int((~valid).sum()), "zero_quantity_rows": int((clean.Quantity == 0).sum()), | |
| "domain": "unspecified user-provided retail", "currency": None, | |
| "schema_reference": UCI_CITATION, | |
| "source_identity_verified": False, | |
| "note": "Schema compatibility does not verify UCI source identity; zeros are not stockout labels."} | |
| receipt_path = path.with_suffix(path.suffix + ".source.json") | |
| if receipt_path.exists(): | |
| receipt = json.loads(receipt_path.read_text()) | |
| if (receipt.get("source_url") == UCI_URL | |
| and receipt.get("dataset_sha256") == metadata["dataset_sha256"] == UCI_WORKBOOK_SHA256 | |
| and receipt.get("archive_sha256") == UCI_ARCHIVE_SHA256): | |
| metadata.update(source_kind="public_dataset", source_identity_verified=True, | |
| domain="UK online retail", currency="GBP", citation=UCI_CITATION, | |
| note="Pinned official UCI workbook identity; zero recorded sales are not stockout labels.", | |
| download_receipt=receipt) | |
| return sales, returns, metadata | |
| def _features(history: np.ndarray, day: pd.Timestamp) -> list[float]: | |
| return [float(history[-1]), float(history[-7]), float(history[-7:].mean()), | |
| float(history[-28:].mean()), float(day.dayofweek)] | |
| def _series(sales: pd.DataFrame, max_items: int) -> tuple[dict[str, pd.Series], pd.DatetimeIndex]: | |
| if sales.empty: | |
| return {}, pd.DatetimeIndex([]) | |
| dates = pd.date_range(sales.day.min(), sales.day.max(), freq="D") | |
| # Rank products only by the initial training interval, never validation/test sales. | |
| initial_end = dates[max(0, len(dates) - 4 * 28 - 1)] | |
| ranking = sales[sales.day <= initial_end].groupby("StockCode").Quantity.sum() | |
| codes = sorted(ranking.index, key=lambda code: (-ranking.loc[code], code))[:max_items] | |
| result = {} | |
| for code in codes: | |
| aggregate = sales[sales.StockCode == code].groupby("day").Quantity.sum() | |
| result[code] = aggregate.reindex(dates, fill_value=0).astype(float) | |
| return result, dates | |
| def _lightgbm(): | |
| try: | |
| import lightgbm as lgb | |
| except ImportError: | |
| return None | |
| return lgb.LGBMRegressor(n_estimators=80, num_leaves=15, learning_rate=0.05, | |
| random_state=42, n_jobs=1, deterministic=True, | |
| force_col_wise=True, verbosity=-1) | |
| def _fit(series: dict, dates, end: int, algorithm: str): | |
| if algorithm != "lightgbm": | |
| return {"algorithm": algorithm} | |
| estimator = _lightgbm() | |
| if estimator is None: | |
| raise ValueError("LightGBM unavailable; no substitute model is reported as LightGBM") | |
| features, targets = [], [] | |
| for values in series.values(): | |
| vector = values.to_numpy() | |
| for index in range(28, end): | |
| features.append(_features(vector[:index], dates[index])) | |
| targets.append(float(vector[index])) | |
| estimator.fit(pd.DataFrame(features, columns=FEATURES), np.asarray(targets)) | |
| return {"algorithm": algorithm, "estimator": estimator} | |
| def _walk_forward(model, series: dict, dates, start: int, end: int): | |
| actual, prediction, feature_rows = [], [], [] | |
| for values in series.values(): | |
| vector = values.to_numpy() | |
| for index in range(start, end): | |
| history = vector[:index] # One-step evaluation: preceding observed days are available. | |
| actual.append(float(vector[index])) | |
| if model["algorithm"] == "seasonal7": | |
| value = history[-7] | |
| elif model["algorithm"] == "rolling28": | |
| value = history[-28:].mean() | |
| else: | |
| feature_rows.append(_features(history, dates[index])) | |
| continue | |
| prediction.append(max(0.0, float(value))) | |
| if feature_rows: | |
| prediction = np.maximum(0, model["estimator"].predict(pd.DataFrame(feature_rows, columns=FEATURES))) | |
| return metrics(actual, prediction) | |
| def train_candidate(payload: dict, artifacts_dir: str | Path | None = None) -> dict: | |
| """Freeze selection on three rolling validation windows, then evaluate holdout once.""" | |
| if not payload.get("input_path"): | |
| return {"status": "INSUFFICIENT_DATA", "reason": "Provide an explicit retail CSV or workbook input_path"} | |
| quality_max_mae = float(payload["quality_max_mae"]) if "quality_max_mae" in payload else None | |
| minimum_relative_improvement = float(payload.get("minimum_relative_improvement", 0.0)) | |
| if quality_max_mae is not None and (not math.isfinite(quality_max_mae) or quality_max_mae < 0): | |
| raise ValueError("quality_max_mae must be finite and nonnegative") | |
| if not math.isfinite(minimum_relative_improvement) or not 0 <= minimum_relative_improvement <= 1: | |
| raise ValueError("minimum_relative_improvement must be finite and between 0 and 1") | |
| root = _root(artifacts_dir) | |
| sales, returns, provenance = read_retail(payload["input_path"]) | |
| # Provenance is derived from ingestion and verified download receipts. | |
| # A request cannot relabel an arbitrary input as an authenticated public dataset. | |
| max_items = int(payload.get("max_items", 50)) | |
| if not 1 <= max_items <= 500: | |
| raise ValueError("max_items must be between 1 and 500") | |
| series, dates = _series(sales, max_items) | |
| horizon = 28 | |
| if len(dates) < 168 or not series: | |
| return {"status": "INSUFFICIENT_DATA", "reason": "Require 168 calendar days and positive initial-training sales", | |
| "provenance": provenance} | |
| final_start = len(dates) - horizon | |
| algorithms = ["seasonal7", "rolling28"] | |
| availability = "available" if _lightgbm() is not None else "unavailable: install lightgbm" | |
| if availability == "available": | |
| algorithms.append("lightgbm") | |
| validation = {} | |
| for algorithm in algorithms: | |
| windows = [] | |
| for start in (final_start - 3 * horizon, final_start - 2 * horizon, final_start - horizon): | |
| fitted = _fit(series, dates, start, algorithm) | |
| windows.append({"train_end": str(dates[start-1].date()), | |
| "validation_start": str(dates[start].date()), | |
| "validation_end": str(dates[start+horizon-1].date()), | |
| **_walk_forward(fitted, series, dates, start, start+horizon)}) | |
| validation[algorithm] = {"windows": windows, "mean_mae": float(np.mean([w["mae"] for w in windows]))} | |
| winner = min(algorithms, key=lambda algorithm: (validation[algorithm]["mean_mae"], algorithm)) | |
| config = {"horizon_days": horizon, "validation_windows": 3, "max_items": max_items, | |
| "selection_metric": "mean_validation_mae", "seed": 42, | |
| "evaluation": "one_step_walk_forward_with_observed_history", "features": list(FEATURES), | |
| "quality_max_mae": quality_max_mae, | |
| "minimum_relative_improvement": minimum_relative_improvement} | |
| # Freeze model/threshold configuration before inspecting final test labels. | |
| frozen_hash = _hash(_json(config)) | |
| model = _fit(series, dates, final_start, winner) | |
| final_metrics = {algorithm: _walk_forward(model if algorithm == winner else _fit(series, dates, final_start, algorithm), | |
| series, dates, final_start, len(dates)) for algorithm in algorithms} | |
| selected = final_metrics[winner] | |
| baseline = final_metrics["seasonal7"]["mae"] | |
| relative = (baseline - selected["mae"]) / baseline if baseline else 0.0 | |
| gates_configured = quality_max_mae is not None and "minimum_relative_improvement" in payload | |
| eligible = (gates_configured and math.isfinite(selected["mae"]) and selected["mae"] <= config["quality_max_mae"] | |
| and relative >= config["minimum_relative_improvement"]) | |
| model["features"] = list(FEATURES) | |
| model["history"] = {code: values.to_numpy()[-28:].tolist() for code, values in series.items()} | |
| model["history_end"] = str(dates[-1].date()) | |
| # This is the evaluated fitted estimator, not a refit on test targets. | |
| artifact = pickle.dumps(model, protocol=5) | |
| artifact_hash = _hash(artifact) | |
| captured_at = datetime.now(timezone.utc).isoformat() | |
| versions = {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "scipy", "scikit-learn")} | |
| try: | |
| versions["lightgbm"] = importlib.metadata.version("lightgbm") | |
| except importlib.metadata.PackageNotFoundError: | |
| versions["lightgbm"] = "unavailable" | |
| manifest = {"schema_version": 1, "created_at": captured_at, "algorithm": winner, | |
| "runtime": {"python": platform.python_version(), "packages": versions}, | |
| "dataset": provenance, "feature_schema_sha256": _hash(_json(list(FEATURES))), | |
| "configuration": config, "configuration_sha256": frozen_hash, | |
| "artifact_sha256": artifact_hash, "validation": validation, | |
| "cohort": {"product_count": len(series), "stock_codes": list(series), | |
| "selection": "highest_positive_unit_sales_in_initial_training_interval", | |
| "initial_training_end": str(dates[final_start - 3 * horizon - 1].date()), | |
| "first_observed_day": str(dates[0].date())}, | |
| "final_test": {"start": str(dates[final_start].date()), "end": str(dates[-1].date()), | |
| "metrics": final_metrics, "selection_frozen_before_test": True}, | |
| "lightgbm": availability, "promotion_eligible": eligible, "quality_gates_configured": gates_configured, | |
| "promotion_relative_mae_improvement": relative, | |
| "serving_history_includes_observed_holdout": True, | |
| "limitations": ["Public retail forecasting does not establish hidden-demand recovery.", | |
| "No stockout, warehouse inventory, or quick-commerce labels."]} | |
| model_id = _hash(_json(manifest)) | |
| manifest["model_id"] = model_id | |
| destination = root / "models" / model_id | |
| with _locked(root): | |
| destination.mkdir(parents=True, exist_ok=False) | |
| try: | |
| _atomic(destination / "artifact.pkl", artifact) | |
| _atomic(destination / "manifest.json", _json(manifest)) | |
| except BaseException: | |
| # Incomplete directories are ignored by list_models and cannot be promoted. | |
| raise | |
| return {"status": "EVALUATED" if eligible or not gates_configured else "REJECTED", "manifest": manifest, | |
| "reason": "quality_gates_not_configured" if not gates_configured else (None if eligible else "configured_quality_gate_failed")} | |
| def _load_manifest(root: Path, model_id: str) -> tuple[dict, bytes]: | |
| if len(model_id) != 64 or any(c not in "0123456789abcdef" for c in model_id): | |
| raise ValueError("Invalid model identifier") | |
| directory = root / "models" / model_id | |
| manifest = json.loads((directory / "manifest.json").read_text()) | |
| content = (directory / "artifact.pkl").read_bytes() | |
| unhashed = {key: value for key, value in manifest.items() if key != "model_id"} | |
| if manifest.get("model_id") != model_id or _hash(_json(unhashed)) != model_id: | |
| raise ValueError("Manifest hash mismatch") | |
| if _hash(content) != manifest["artifact_sha256"]: | |
| raise ValueError("Artifact hash mismatch") | |
| return manifest, content | |
| def list_models(artifacts_dir=None) -> list: | |
| root = _root(artifacts_dir) | |
| result = [] | |
| for path in sorted((root / "models").glob("*/manifest.json")): | |
| try: | |
| result.append(_load_manifest(root, path.parent.name)[0]) | |
| except (ValueError, OSError, KeyError): | |
| continue | |
| return result | |
| def _verify_serving(model: dict): | |
| algorithm = model.get("algorithm") | |
| if algorithm not in {"seasonal7", "rolling28", "lightgbm"}: | |
| raise ValueError("Unsupported artifact algorithm") | |
| for history in model["history"].values(): | |
| if len(history) < 28 or not np.isfinite(history).all(): | |
| raise ValueError("Artifact has invalid serving history") | |
| if algorithm == "lightgbm": | |
| check = model["estimator"].predict(pd.DataFrame(np.zeros((1, len(FEATURES))), columns=FEATURES)) | |
| if not np.isfinite(check).all(): | |
| raise ValueError("Artifact failed serving verification") | |
| def promote_model(model_id: str, artifacts_dir=None) -> dict: | |
| root = _root(artifacts_dir) | |
| with _locked(root): | |
| manifest, content = _load_manifest(root, model_id) | |
| if not manifest.get("quality_gates_configured") or not manifest.get("promotion_eligible"): | |
| return {"status": "REJECTED", "reason": "configured_quality_gate_failed", "manifest": manifest} | |
| _verify_serving(pickle.loads(content)) # only registry artifacts this service wrote, hash-verified # nosec B301 | |
| pointer_path = root / "active.json" | |
| previous = json.loads(pointer_path.read_text()) if pointer_path.exists() else {} | |
| if previous.get("active") != model_id: | |
| _atomic(pointer_path, _json({"active": model_id, "previous": previous.get("active")})) | |
| return {"status": "PROMOTED", "manifest": manifest} | |
| def rollback_model(artifacts_dir=None) -> dict: | |
| root = _root(artifacts_dir) | |
| with _locked(root): | |
| path = root / "active.json" | |
| pointer = json.loads(path.read_text()) if path.exists() else {} | |
| previous = pointer.get("previous") | |
| if not previous: | |
| return {"status": "REJECTED", "reason": "no_previous_model"} | |
| manifest, content = _load_manifest(root, previous) | |
| _verify_serving(pickle.loads(content)) # hash-verified local registry artifact # nosec B301 | |
| _atomic(path, _json({"active": previous, "previous": pointer["active"]})) | |
| return {"status": "PROMOTED", "rollback": True, "manifest": manifest} | |
| def predict_rows(rows: list[dict], artifacts_dir=None) -> dict: | |
| root = _root(artifacts_dir) | |
| path = root / "active.json" | |
| if not path.exists(): | |
| return {"status": "UNAVAILABLE", "reason": "no_active_model", "predictions": []} | |
| model_id = json.loads(path.read_text())["active"] | |
| # Bind this entire request to one immutable model even if promotion happens concurrently. | |
| manifest, content = _load_manifest(root, model_id) | |
| with _registry_lock: | |
| cached = _model_cache.get(str(root)) | |
| if not cached or cached[0] != model_id: | |
| model = pickle.loads(content) # hash-verified local registry artifact # nosec B301 | |
| _verify_serving(model) | |
| _model_cache[str(root)] = (model_id, model) | |
| else: | |
| model = cached[1] | |
| predictions = [] | |
| for row in rows: | |
| code = str(row["stock_code"]) | |
| history = np.asarray(row.get("history", model["history"].get(code, [])), float) | |
| if len(history) < 28 or not np.isfinite(history).all() or (history < 0).any(): | |
| raise ValueError("Each row requires at least 28 finite nonnegative observed sales values") | |
| day = pd.Timestamp(row.get("date", pd.Timestamp(model["history_end"]) + pd.Timedelta(days=1))) | |
| if pd.isna(day): | |
| raise ValueError("Forecast date must be a valid calendar date") | |
| if row.get("date") and day.normalize() != pd.Timestamp(model["history_end"]) + pd.Timedelta(days=1) and "history" not in row: | |
| raise ValueError("Other forecast dates require explicit observed history ending immediately before the forecast") | |
| if model["algorithm"] == "seasonal7": | |
| estimate = history[-7] | |
| elif model["algorithm"] == "rolling28": | |
| estimate = history[-28:].mean() | |
| else: | |
| estimate = model["estimator"].predict(pd.DataFrame([_features(history, day)], columns=FEATURES))[0] | |
| if not np.isfinite(estimate): | |
| raise ValueError("Model returned a non-finite prediction") | |
| predictions.append({"stock_code": code, "date": str(day.date()), "positive_sales_units": max(0.0, float(estimate))}) | |
| return {"status": "AVAILABLE", "model_id": model_id, "artifact_sha256": manifest["artifact_sha256"], | |
| "source_kind": manifest["dataset"]["source_kind"], "predictions": predictions, | |
| "note": "Positive-sales forecast; not warehouse inventory or latent-demand recovery."} | |
| def run_training(payload: dict, artifacts_dir=None) -> dict: | |
| try: | |
| candidate = train_candidate(payload, artifacts_dir) | |
| if candidate["status"] == "EVALUATED" and payload.get("auto_promote") is True: | |
| return promote_model(candidate["manifest"]["model_id"], artifacts_dir) | |
| return candidate | |
| except Exception as exc: | |
| return {"status": "FAILED", "reason": f"{type(exc).__name__}: {exc}"} | |
| def train_job(payload: dict) -> dict: | |
| return run_training(payload) | |
| def run_retail_study(input_path, output_dir, *, max_items=50, source_kind="user_input") -> dict: | |
| result = train_candidate({"input_path": str(input_path), "max_items": max_items, "source_kind": source_kind}, output_dir) | |
| _atomic(Path(output_dir) / "retail_study.json", _json(result)) | |
| return result | |
| def download_uci(output_path: str | Path) -> Path: | |
| """Explicit command-line opt-in only. Never called by training jobs.""" | |
| import urllib.request | |
| with urllib.request.urlopen(UCI_URL, timeout=120) as response: # constant https URL # nosec B310 | |
| content = response.read(100_000_001) | |
| if len(content) > 100_000_000: | |
| raise ValueError("UCI archive exceeds 100 MB limit") | |
| if _hash(content) != UCI_ARCHIVE_SHA256: | |
| raise ValueError("UCI archive differs from the pinned study source; verify the upstream release before updating its identity") | |
| with zipfile.ZipFile(io.BytesIO(content)) as archive: | |
| candidates = [name for name in archive.namelist() if name.lower().endswith(".xlsx")] | |
| if len(candidates) != 1 or archive.getinfo(candidates[0]).file_size > 100_000_000: | |
| raise ValueError("Unexpected UCI archive contents") | |
| workbook = archive.read(candidates[0]) | |
| if _hash(workbook) != UCI_WORKBOOK_SHA256: | |
| raise ValueError("UCI workbook differs from the pinned study source") | |
| _atomic(Path(output_path), workbook) | |
| _atomic(Path(output_path).with_suffix(Path(output_path).suffix + ".source.json"), | |
| _json({"source_url": UCI_URL, "dataset_sha256": _hash(workbook), | |
| "archive_sha256": _hash(content), "citation": UCI_CITATION, | |
| "captured_at": datetime.now(timezone.utc).isoformat()})) | |
| return Path(output_path) | |
| def run_censoring_study(output_dir, *, seeds=(7, 19, 41), levels=(0.2, 0.4, 0.6), n_samples=1200) -> dict: | |
| """Controlled Gaussian censoring experiment; latent test targets are known. | |
| Uses prediction intervals for latent outcomes, not parameter confidence bands. | |
| Thresholds are fixed using training latent values; test labels never fit the model. | |
| """ | |
| from scipy.stats import norm | |
| from backend.ml.censored_demand import TobitRegressor | |
| if n_samples < 200 or not seeds or not levels or any(not 0 < level < 0.9 for level in levels): | |
| raise ValueError("Require >=200 samples, seeds, and censoring levels in (0, 0.9)") | |
| results = [] | |
| for seed in seeds: | |
| rng = np.random.default_rng(int(seed)) | |
| features = rng.normal(size=(n_samples, 2)) | |
| latent = 30 + features @ np.array([4.0, -2.0]) + rng.normal(0, 3, n_samples) | |
| split = int(n_samples * .7) | |
| for level in levels: | |
| cap = float(np.quantile(latent[:split], 1 - level)) | |
| observed = np.minimum(latent, cap) | |
| censored = latent >= cap | |
| model = TobitRegressor().fit(features[:split], observed[:split], censored[:split]) | |
| mean = model.predict_latent(features[split:]) | |
| sigma = model.get_dynamic_sigma(features[split:]) | |
| truth = latent[split:] | |
| test_censored = censored[split:] | |
| recovered = model.impute_demand(features[split:], observed[split:], test_censored, sample=False) | |
| naive = np.column_stack([np.ones(split), features[:split]]) | |
| coefficients = np.linalg.lstsq(naive, observed[:split], rcond=None)[0] | |
| baseline = np.column_stack([np.ones(n_samples-split), features[split:]]) @ coefficients | |
| coverage = float(np.mean((truth >= mean - norm.ppf(.95)*sigma) & (truth <= mean + norm.ppf(.95)*sigma))) | |
| results.append({"seed": int(seed), "target_censoring": float(level), "threshold": cap, | |
| "observed_train_censoring": float(censored[:split].mean()), | |
| "test_censored_count": int(test_censored.sum()), | |
| "tobit_latent_prediction": metrics(truth, mean), | |
| "naive_observed_ols": metrics(truth, baseline), | |
| "censored_recovery": metrics(truth[test_censored], recovered[test_censored]) if test_censored.any() else None, | |
| "censored_observed_baseline": metrics(truth[test_censored], observed[split:][test_censored]) if test_censored.any() else None, | |
| "censored_naive_ols_baseline": metrics(truth[test_censored], baseline[test_censored]) if test_censored.any() else None, | |
| "latent_prediction_interval_90_coverage": coverage, | |
| "converged": bool(getattr(model, "optimization_success_", False)), | |
| "optimization_message": str(getattr(model, "optimization_message_", "not_reported"))}) | |
| report = {"source_kind": "simulation", "scenario": "known_gaussian_latent_right_censoring", | |
| "n_samples": n_samples, "train_fraction": .7, "runs": results, | |
| "mean_interval_90_coverage": float(np.mean([run["latent_prediction_interval_90_coverage"] for run in results])), | |
| "limitations": ["Controlled synthetic experiment; not recovered real sales.", | |
| "Prediction intervals assume the fitted Gaussian model; no claim of real inventory observations."]} | |
| _atomic(Path(output_dir) / "censoring_study.json", _json(report)) | |
| return report | |