"""Reproducible offline studies and an immutable, local model registry. No imports from the application, database, credentials, or provider clients. A public-data study is not evidence of quick-commerce demand or hidden sales. """ from __future__ import annotations from contextlib import contextmanager from datetime import datetime, timezone import hashlib import io import importlib.metadata import platform import json import math import os from pathlib import Path import pickle import tempfile import threading import zipfile from typing import Any import numpy as np import pandas as pd FEATURES = ("lag_1", "lag_7", "mean_7", "mean_28", "weekday") UCI_URL = "https://archive.ics.uci.edu/static/public/502/online+retail+ii.zip" UCI_CITATION = "Chen, D. (2012). Online Retail II. https://doi.org/10.24432/C5CG6D (CC BY 4.0)" # Pinned from the official HTTPS download used for the reproducible study. # A caller-written sidecar alone cannot establish public dataset identity. UCI_WORKBOOK_SHA256 = "bcbe73b35f5b7babf197fb0cb983a11f5d9ff929078d4aa53d171b1f2df2e980" UCI_ARCHIVE_SHA256 = "572e36277c2390fbfde10664750731e0a86f55e33470d91919085f0408e67bfb" _registry_lock = threading.RLock() _model_cache: dict[str, tuple[str, Any]] = {} def _json(value: Any) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() def _hash(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def _atomic(path: Path, data: bytes) -> None: path.parent.mkdir(parents=True, exist_ok=True) descriptor, name = tempfile.mkstemp(prefix=".pending-", dir=path.parent) try: with os.fdopen(descriptor, "wb") as stream: stream.write(data) stream.flush() os.fsync(stream.fileno()) os.replace(name, path) finally: if os.path.exists(name): os.unlink(name) def _root(artifacts_dir: str | Path | None = None) -> Path: return Path(artifacts_dir or os.environ.get("HYPERFLOW_ARTIFACT_DIR", ".local/artifacts")).resolve() @contextmanager def _locked(root: Path): """Serialize registry pointer writes across threads and worker processes.""" import fcntl root.mkdir(parents=True, exist_ok=True) with _registry_lock, (root / ".registry.lock").open("a+b") as stream: fcntl.flock(stream, fcntl.LOCK_EX) try: yield finally: fcntl.flock(stream, fcntl.LOCK_UN) def metrics(actual, predicted) -> dict: actual, predicted = np.asarray(actual, float), np.asarray(predicted, float) if actual.size == 0 or actual.shape != predicted.shape: raise ValueError("Metric arrays must have equal, non-empty shapes") if not np.isfinite(actual).all() or not np.isfinite(predicted).all(): raise ValueError("Metrics require finite inputs") error = np.abs(actual - predicted) denominator = float(np.abs(actual).sum()) return {"mae": float(error.mean()), "rmse": float(np.sqrt(np.mean(error ** 2))), "wmape": float(error.sum() / denominator) if denominator else None, "observations": int(actual.size)} def read_retail(input_path: str | Path) -> tuple[pd.DataFrame, pd.DataFrame, dict]: """Read both original workbook sheets, or an explicitly supplied CSV. Cancellation invoices and negative quantities are separated before training. Missing customer IDs do not invalidate a product/day forecasting observation. Zero sales mean no recorded positive sales, never proven stockouts. """ path = Path(input_path) # Parse and fingerprint the same byte snapshot even if an input is replaced # while ingestion is running. input_bytes = path.read_bytes() if path.suffix.lower() == ".csv": frame = pd.read_csv(io.BytesIO(input_bytes)) elif path.suffix.lower() in {".xlsx", ".xls"}: try: frame = pd.concat(pd.read_excel(io.BytesIO(input_bytes), sheet_name=None).values(), ignore_index=True) except ImportError as exc: raise ValueError("Reading .xlsx requires openpyxl; CSV input remains available") from exc else: raise ValueError("Input must be .csv or .xlsx/.xls") aliases = {"invoice": "InvoiceNo", "invoiceno": "InvoiceNo", "stockcode": "StockCode", "quantity": "Quantity", "invoicedate": "InvoiceDate", "price": "UnitPrice", "unitprice": "UnitPrice", "country": "Country"} frame = frame.rename(columns={column: aliases.get(str(column).lower().replace(" ", ""), column) for column in frame.columns}) required = {"InvoiceNo", "StockCode", "Quantity", "InvoiceDate", "UnitPrice"} if not required.issubset(frame.columns): raise ValueError(f"Missing required retail columns: {sorted(required - set(frame.columns))}") frame["Quantity"] = pd.to_numeric(frame.Quantity, errors="coerce") frame["UnitPrice"] = pd.to_numeric(frame.UnitPrice, errors="coerce") frame["InvoiceDate"] = pd.to_datetime(frame.InvoiceDate, errors="coerce", format="mixed") valid = (frame.InvoiceDate.notna() & frame.StockCode.notna() & frame.InvoiceNo.notna() & np.isfinite(frame.Quantity) & np.isfinite(frame.UnitPrice) & (frame.UnitPrice >= 0)) clean = frame.loc[valid].copy() cancellation = clean.InvoiceNo.astype(str).str.upper().str.startswith("C") | (clean.Quantity < 0) returns = clean.loc[cancellation].copy() sales = clean.loc[~cancellation & (clean.Quantity > 0)].copy() sales["StockCode"] = sales.StockCode.astype(str) sales["day"] = sales.InvoiceDate.dt.normalize() # Unknown uploaded data does not inherit the original dataset's domain/currency. metadata = {"source_kind": "user_input", "source_format": "uci_online_retail_ii_schema", "dataset_sha256": _hash(input_bytes), "input_rows": len(frame), "positive_sales_rows": len(sales), "return_cancellation_rows": len(returns), "invalid_rows": int((~valid).sum()), "zero_quantity_rows": int((clean.Quantity == 0).sum()), "domain": "unspecified user-provided retail", "currency": None, "schema_reference": UCI_CITATION, "source_identity_verified": False, "note": "Schema compatibility does not verify UCI source identity; zeros are not stockout labels."} receipt_path = path.with_suffix(path.suffix + ".source.json") if receipt_path.exists(): receipt = json.loads(receipt_path.read_text()) if (receipt.get("source_url") == UCI_URL and receipt.get("dataset_sha256") == metadata["dataset_sha256"] == UCI_WORKBOOK_SHA256 and receipt.get("archive_sha256") == UCI_ARCHIVE_SHA256): metadata.update(source_kind="public_dataset", source_identity_verified=True, domain="UK online retail", currency="GBP", citation=UCI_CITATION, note="Pinned official UCI workbook identity; zero recorded sales are not stockout labels.", download_receipt=receipt) return sales, returns, metadata def _features(history: np.ndarray, day: pd.Timestamp) -> list[float]: return [float(history[-1]), float(history[-7]), float(history[-7:].mean()), float(history[-28:].mean()), float(day.dayofweek)] def _series(sales: pd.DataFrame, max_items: int) -> tuple[dict[str, pd.Series], pd.DatetimeIndex]: if sales.empty: return {}, pd.DatetimeIndex([]) dates = pd.date_range(sales.day.min(), sales.day.max(), freq="D") # Rank products only by the initial training interval, never validation/test sales. initial_end = dates[max(0, len(dates) - 4 * 28 - 1)] ranking = sales[sales.day <= initial_end].groupby("StockCode").Quantity.sum() codes = sorted(ranking.index, key=lambda code: (-ranking.loc[code], code))[:max_items] result = {} for code in codes: aggregate = sales[sales.StockCode == code].groupby("day").Quantity.sum() result[code] = aggregate.reindex(dates, fill_value=0).astype(float) return result, dates def _lightgbm(): try: import lightgbm as lgb except ImportError: return None return lgb.LGBMRegressor(n_estimators=80, num_leaves=15, learning_rate=0.05, random_state=42, n_jobs=1, deterministic=True, force_col_wise=True, verbosity=-1) def _fit(series: dict, dates, end: int, algorithm: str): if algorithm != "lightgbm": return {"algorithm": algorithm} estimator = _lightgbm() if estimator is None: raise ValueError("LightGBM unavailable; no substitute model is reported as LightGBM") features, targets = [], [] for values in series.values(): vector = values.to_numpy() for index in range(28, end): features.append(_features(vector[:index], dates[index])) targets.append(float(vector[index])) estimator.fit(pd.DataFrame(features, columns=FEATURES), np.asarray(targets)) return {"algorithm": algorithm, "estimator": estimator} def _walk_forward(model, series: dict, dates, start: int, end: int): actual, prediction, feature_rows = [], [], [] for values in series.values(): vector = values.to_numpy() for index in range(start, end): history = vector[:index] # One-step evaluation: preceding observed days are available. actual.append(float(vector[index])) if model["algorithm"] == "seasonal7": value = history[-7] elif model["algorithm"] == "rolling28": value = history[-28:].mean() else: feature_rows.append(_features(history, dates[index])) continue prediction.append(max(0.0, float(value))) if feature_rows: prediction = np.maximum(0, model["estimator"].predict(pd.DataFrame(feature_rows, columns=FEATURES))) return metrics(actual, prediction) def train_candidate(payload: dict, artifacts_dir: str | Path | None = None) -> dict: """Freeze selection on three rolling validation windows, then evaluate holdout once.""" if not payload.get("input_path"): return {"status": "INSUFFICIENT_DATA", "reason": "Provide an explicit retail CSV or workbook input_path"} quality_max_mae = float(payload["quality_max_mae"]) if "quality_max_mae" in payload else None minimum_relative_improvement = float(payload.get("minimum_relative_improvement", 0.0)) if quality_max_mae is not None and (not math.isfinite(quality_max_mae) or quality_max_mae < 0): raise ValueError("quality_max_mae must be finite and nonnegative") if not math.isfinite(minimum_relative_improvement) or not 0 <= minimum_relative_improvement <= 1: raise ValueError("minimum_relative_improvement must be finite and between 0 and 1") root = _root(artifacts_dir) sales, returns, provenance = read_retail(payload["input_path"]) # Provenance is derived from ingestion and verified download receipts. # A request cannot relabel an arbitrary input as an authenticated public dataset. max_items = int(payload.get("max_items", 50)) if not 1 <= max_items <= 500: raise ValueError("max_items must be between 1 and 500") series, dates = _series(sales, max_items) horizon = 28 if len(dates) < 168 or not series: return {"status": "INSUFFICIENT_DATA", "reason": "Require 168 calendar days and positive initial-training sales", "provenance": provenance} final_start = len(dates) - horizon algorithms = ["seasonal7", "rolling28"] availability = "available" if _lightgbm() is not None else "unavailable: install lightgbm" if availability == "available": algorithms.append("lightgbm") validation = {} for algorithm in algorithms: windows = [] for start in (final_start - 3 * horizon, final_start - 2 * horizon, final_start - horizon): fitted = _fit(series, dates, start, algorithm) windows.append({"train_end": str(dates[start-1].date()), "validation_start": str(dates[start].date()), "validation_end": str(dates[start+horizon-1].date()), **_walk_forward(fitted, series, dates, start, start+horizon)}) validation[algorithm] = {"windows": windows, "mean_mae": float(np.mean([w["mae"] for w in windows]))} winner = min(algorithms, key=lambda algorithm: (validation[algorithm]["mean_mae"], algorithm)) config = {"horizon_days": horizon, "validation_windows": 3, "max_items": max_items, "selection_metric": "mean_validation_mae", "seed": 42, "evaluation": "one_step_walk_forward_with_observed_history", "features": list(FEATURES), "quality_max_mae": quality_max_mae, "minimum_relative_improvement": minimum_relative_improvement} # Freeze model/threshold configuration before inspecting final test labels. frozen_hash = _hash(_json(config)) model = _fit(series, dates, final_start, winner) final_metrics = {algorithm: _walk_forward(model if algorithm == winner else _fit(series, dates, final_start, algorithm), series, dates, final_start, len(dates)) for algorithm in algorithms} selected = final_metrics[winner] baseline = final_metrics["seasonal7"]["mae"] relative = (baseline - selected["mae"]) / baseline if baseline else 0.0 gates_configured = quality_max_mae is not None and "minimum_relative_improvement" in payload eligible = (gates_configured and math.isfinite(selected["mae"]) and selected["mae"] <= config["quality_max_mae"] and relative >= config["minimum_relative_improvement"]) model["features"] = list(FEATURES) model["history"] = {code: values.to_numpy()[-28:].tolist() for code, values in series.items()} model["history_end"] = str(dates[-1].date()) # This is the evaluated fitted estimator, not a refit on test targets. artifact = pickle.dumps(model, protocol=5) artifact_hash = _hash(artifact) captured_at = datetime.now(timezone.utc).isoformat() versions = {name: importlib.metadata.version(name) for name in ("numpy", "pandas", "scipy", "scikit-learn")} try: versions["lightgbm"] = importlib.metadata.version("lightgbm") except importlib.metadata.PackageNotFoundError: versions["lightgbm"] = "unavailable" manifest = {"schema_version": 1, "created_at": captured_at, "algorithm": winner, "runtime": {"python": platform.python_version(), "packages": versions}, "dataset": provenance, "feature_schema_sha256": _hash(_json(list(FEATURES))), "configuration": config, "configuration_sha256": frozen_hash, "artifact_sha256": artifact_hash, "validation": validation, "cohort": {"product_count": len(series), "stock_codes": list(series), "selection": "highest_positive_unit_sales_in_initial_training_interval", "initial_training_end": str(dates[final_start - 3 * horizon - 1].date()), "first_observed_day": str(dates[0].date())}, "final_test": {"start": str(dates[final_start].date()), "end": str(dates[-1].date()), "metrics": final_metrics, "selection_frozen_before_test": True}, "lightgbm": availability, "promotion_eligible": eligible, "quality_gates_configured": gates_configured, "promotion_relative_mae_improvement": relative, "serving_history_includes_observed_holdout": True, "limitations": ["Public retail forecasting does not establish hidden-demand recovery.", "No stockout, warehouse inventory, or quick-commerce labels."]} model_id = _hash(_json(manifest)) manifest["model_id"] = model_id destination = root / "models" / model_id with _locked(root): destination.mkdir(parents=True, exist_ok=False) try: _atomic(destination / "artifact.pkl", artifact) _atomic(destination / "manifest.json", _json(manifest)) except BaseException: # Incomplete directories are ignored by list_models and cannot be promoted. raise return {"status": "EVALUATED" if eligible or not gates_configured else "REJECTED", "manifest": manifest, "reason": "quality_gates_not_configured" if not gates_configured else (None if eligible else "configured_quality_gate_failed")} def _load_manifest(root: Path, model_id: str) -> tuple[dict, bytes]: if len(model_id) != 64 or any(c not in "0123456789abcdef" for c in model_id): raise ValueError("Invalid model identifier") directory = root / "models" / model_id manifest = json.loads((directory / "manifest.json").read_text()) content = (directory / "artifact.pkl").read_bytes() unhashed = {key: value for key, value in manifest.items() if key != "model_id"} if manifest.get("model_id") != model_id or _hash(_json(unhashed)) != model_id: raise ValueError("Manifest hash mismatch") if _hash(content) != manifest["artifact_sha256"]: raise ValueError("Artifact hash mismatch") return manifest, content def list_models(artifacts_dir=None) -> list: root = _root(artifacts_dir) result = [] for path in sorted((root / "models").glob("*/manifest.json")): try: result.append(_load_manifest(root, path.parent.name)[0]) except (ValueError, OSError, KeyError): continue return result def _verify_serving(model: dict): algorithm = model.get("algorithm") if algorithm not in {"seasonal7", "rolling28", "lightgbm"}: raise ValueError("Unsupported artifact algorithm") for history in model["history"].values(): if len(history) < 28 or not np.isfinite(history).all(): raise ValueError("Artifact has invalid serving history") if algorithm == "lightgbm": check = model["estimator"].predict(pd.DataFrame(np.zeros((1, len(FEATURES))), columns=FEATURES)) if not np.isfinite(check).all(): raise ValueError("Artifact failed serving verification") def promote_model(model_id: str, artifacts_dir=None) -> dict: root = _root(artifacts_dir) with _locked(root): manifest, content = _load_manifest(root, model_id) if not manifest.get("quality_gates_configured") or not manifest.get("promotion_eligible"): return {"status": "REJECTED", "reason": "configured_quality_gate_failed", "manifest": manifest} _verify_serving(pickle.loads(content)) # only registry artifacts this service wrote, hash-verified # nosec B301 pointer_path = root / "active.json" previous = json.loads(pointer_path.read_text()) if pointer_path.exists() else {} if previous.get("active") != model_id: _atomic(pointer_path, _json({"active": model_id, "previous": previous.get("active")})) return {"status": "PROMOTED", "manifest": manifest} def rollback_model(artifacts_dir=None) -> dict: root = _root(artifacts_dir) with _locked(root): path = root / "active.json" pointer = json.loads(path.read_text()) if path.exists() else {} previous = pointer.get("previous") if not previous: return {"status": "REJECTED", "reason": "no_previous_model"} manifest, content = _load_manifest(root, previous) _verify_serving(pickle.loads(content)) # hash-verified local registry artifact # nosec B301 _atomic(path, _json({"active": previous, "previous": pointer["active"]})) return {"status": "PROMOTED", "rollback": True, "manifest": manifest} def predict_rows(rows: list[dict], artifacts_dir=None) -> dict: root = _root(artifacts_dir) path = root / "active.json" if not path.exists(): return {"status": "UNAVAILABLE", "reason": "no_active_model", "predictions": []} model_id = json.loads(path.read_text())["active"] # Bind this entire request to one immutable model even if promotion happens concurrently. manifest, content = _load_manifest(root, model_id) with _registry_lock: cached = _model_cache.get(str(root)) if not cached or cached[0] != model_id: model = pickle.loads(content) # hash-verified local registry artifact # nosec B301 _verify_serving(model) _model_cache[str(root)] = (model_id, model) else: model = cached[1] predictions = [] for row in rows: code = str(row["stock_code"]) history = np.asarray(row.get("history", model["history"].get(code, [])), float) if len(history) < 28 or not np.isfinite(history).all() or (history < 0).any(): raise ValueError("Each row requires at least 28 finite nonnegative observed sales values") day = pd.Timestamp(row.get("date", pd.Timestamp(model["history_end"]) + pd.Timedelta(days=1))) if pd.isna(day): raise ValueError("Forecast date must be a valid calendar date") if row.get("date") and day.normalize() != pd.Timestamp(model["history_end"]) + pd.Timedelta(days=1) and "history" not in row: raise ValueError("Other forecast dates require explicit observed history ending immediately before the forecast") if model["algorithm"] == "seasonal7": estimate = history[-7] elif model["algorithm"] == "rolling28": estimate = history[-28:].mean() else: estimate = model["estimator"].predict(pd.DataFrame([_features(history, day)], columns=FEATURES))[0] if not np.isfinite(estimate): raise ValueError("Model returned a non-finite prediction") predictions.append({"stock_code": code, "date": str(day.date()), "positive_sales_units": max(0.0, float(estimate))}) return {"status": "AVAILABLE", "model_id": model_id, "artifact_sha256": manifest["artifact_sha256"], "source_kind": manifest["dataset"]["source_kind"], "predictions": predictions, "note": "Positive-sales forecast; not warehouse inventory or latent-demand recovery."} def run_training(payload: dict, artifacts_dir=None) -> dict: try: candidate = train_candidate(payload, artifacts_dir) if candidate["status"] == "EVALUATED" and payload.get("auto_promote") is True: return promote_model(candidate["manifest"]["model_id"], artifacts_dir) return candidate except Exception as exc: return {"status": "FAILED", "reason": f"{type(exc).__name__}: {exc}"} def train_job(payload: dict) -> dict: return run_training(payload) def run_retail_study(input_path, output_dir, *, max_items=50, source_kind="user_input") -> dict: result = train_candidate({"input_path": str(input_path), "max_items": max_items, "source_kind": source_kind}, output_dir) _atomic(Path(output_dir) / "retail_study.json", _json(result)) return result def download_uci(output_path: str | Path) -> Path: """Explicit command-line opt-in only. Never called by training jobs.""" import urllib.request with urllib.request.urlopen(UCI_URL, timeout=120) as response: # constant https URL # nosec B310 content = response.read(100_000_001) if len(content) > 100_000_000: raise ValueError("UCI archive exceeds 100 MB limit") if _hash(content) != UCI_ARCHIVE_SHA256: raise ValueError("UCI archive differs from the pinned study source; verify the upstream release before updating its identity") with zipfile.ZipFile(io.BytesIO(content)) as archive: candidates = [name for name in archive.namelist() if name.lower().endswith(".xlsx")] if len(candidates) != 1 or archive.getinfo(candidates[0]).file_size > 100_000_000: raise ValueError("Unexpected UCI archive contents") workbook = archive.read(candidates[0]) if _hash(workbook) != UCI_WORKBOOK_SHA256: raise ValueError("UCI workbook differs from the pinned study source") _atomic(Path(output_path), workbook) _atomic(Path(output_path).with_suffix(Path(output_path).suffix + ".source.json"), _json({"source_url": UCI_URL, "dataset_sha256": _hash(workbook), "archive_sha256": _hash(content), "citation": UCI_CITATION, "captured_at": datetime.now(timezone.utc).isoformat()})) return Path(output_path) def run_censoring_study(output_dir, *, seeds=(7, 19, 41), levels=(0.2, 0.4, 0.6), n_samples=1200) -> dict: """Controlled Gaussian censoring experiment; latent test targets are known. Uses prediction intervals for latent outcomes, not parameter confidence bands. Thresholds are fixed using training latent values; test labels never fit the model. """ from scipy.stats import norm from backend.ml.censored_demand import TobitRegressor if n_samples < 200 or not seeds or not levels or any(not 0 < level < 0.9 for level in levels): raise ValueError("Require >=200 samples, seeds, and censoring levels in (0, 0.9)") results = [] for seed in seeds: rng = np.random.default_rng(int(seed)) features = rng.normal(size=(n_samples, 2)) latent = 30 + features @ np.array([4.0, -2.0]) + rng.normal(0, 3, n_samples) split = int(n_samples * .7) for level in levels: cap = float(np.quantile(latent[:split], 1 - level)) observed = np.minimum(latent, cap) censored = latent >= cap model = TobitRegressor().fit(features[:split], observed[:split], censored[:split]) mean = model.predict_latent(features[split:]) sigma = model.get_dynamic_sigma(features[split:]) truth = latent[split:] test_censored = censored[split:] recovered = model.impute_demand(features[split:], observed[split:], test_censored, sample=False) naive = np.column_stack([np.ones(split), features[:split]]) coefficients = np.linalg.lstsq(naive, observed[:split], rcond=None)[0] baseline = np.column_stack([np.ones(n_samples-split), features[split:]]) @ coefficients coverage = float(np.mean((truth >= mean - norm.ppf(.95)*sigma) & (truth <= mean + norm.ppf(.95)*sigma))) results.append({"seed": int(seed), "target_censoring": float(level), "threshold": cap, "observed_train_censoring": float(censored[:split].mean()), "test_censored_count": int(test_censored.sum()), "tobit_latent_prediction": metrics(truth, mean), "naive_observed_ols": metrics(truth, baseline), "censored_recovery": metrics(truth[test_censored], recovered[test_censored]) if test_censored.any() else None, "censored_observed_baseline": metrics(truth[test_censored], observed[split:][test_censored]) if test_censored.any() else None, "censored_naive_ols_baseline": metrics(truth[test_censored], baseline[test_censored]) if test_censored.any() else None, "latent_prediction_interval_90_coverage": coverage, "converged": bool(getattr(model, "optimization_success_", False)), "optimization_message": str(getattr(model, "optimization_message_", "not_reported"))}) report = {"source_kind": "simulation", "scenario": "known_gaussian_latent_right_censoring", "n_samples": n_samples, "train_fraction": .7, "runs": results, "mean_interval_90_coverage": float(np.mean([run["latent_prediction_interval_90_coverage"] for run in results])), "limitations": ["Controlled synthetic experiment; not recovered real sales.", "Prediction intervals assume the fitted Gaussian model; no claim of real inventory observations."]} _atomic(Path(output_dir) / "censoring_study.json", _json(report)) return report