File size: 8,953 Bytes
84f224f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 | from __future__ import annotations
from pathlib import Path
from typing import Any, Iterable
import pandas as pd
TIMESTAMP_NAME_HINTS = ("timestamp", "time", "date", "datetime", "ds")
def _resolve_file_path(file_obj: Any) -> Path:
if file_obj is None:
raise ValueError("No file provided.")
if isinstance(file_obj, (str, Path)):
return Path(file_obj)
if isinstance(file_obj, dict):
maybe_path = file_obj.get("path") or file_obj.get("name")
if maybe_path:
return Path(maybe_path)
maybe_name = getattr(file_obj, "name", None)
if maybe_name:
return Path(maybe_name)
raise ValueError("Unsupported file object type.")
def load_timeseries_from_file(file_obj: Any) -> pd.DataFrame:
file_path = _resolve_file_path(file_obj)
suffix = file_path.suffix.lower()
if suffix == ".csv":
df = pd.read_csv(file_path)
elif suffix in {".xlsx", ".xls"}:
df = pd.read_excel(file_path)
else:
raise ValueError(f"Unsupported file extension: {suffix}")
if df.empty:
raise ValueError("Uploaded file is empty.")
df.columns = [str(col).strip() for col in df.columns]
timestamp_col = detect_timestamp_column(df)
if timestamp_col:
df[timestamp_col] = pd.to_datetime(df[timestamp_col], errors="coerce")
return df
def detect_timestamp_column(df: pd.DataFrame) -> str | None:
for col in df.columns:
if pd.api.types.is_datetime64_any_dtype(df[col]):
return col
for col in df.columns:
normalized = str(col).strip().lower()
if any(hint in normalized for hint in TIMESTAMP_NAME_HINTS):
parsed = pd.to_datetime(df[col], errors="coerce")
if parsed.notna().mean() >= 0.7:
return col
for col in df.columns:
series = df[col]
if pd.api.types.is_numeric_dtype(series):
continue
parsed = pd.to_datetime(series, errors="coerce")
if parsed.notna().mean() >= 0.9:
return col
return None
def detect_numeric_columns(df: pd.DataFrame, exclude: Iterable[str] | None = None) -> list[str]:
excluded = set(exclude or [])
numeric_cols: list[str] = []
for col in df.columns:
if col in excluded:
continue
if pd.api.types.is_numeric_dtype(df[col]):
numeric_cols.append(col)
return numeric_cols
def infer_frequency(df: pd.DataFrame, timestamp_col: str | None) -> str | None:
if not timestamp_col or timestamp_col not in df.columns:
return None
timestamps = pd.to_datetime(df[timestamp_col], errors="coerce").dropna().sort_values()
timestamps = timestamps.drop_duplicates()
if len(timestamps) < 3:
return None
inferred = pd.infer_freq(timestamps)
if inferred:
return inferred
deltas = timestamps.diff().dropna()
if deltas.empty:
return None
mode_delta = deltas.mode()
if mode_delta.empty:
return None
try:
return pd.tseries.frequencies.to_offset(mode_delta.iloc[0]).freqstr
except ValueError:
return None
def validate_timeseries(
df: pd.DataFrame,
timestamp_col: str | None = None,
target_cols: list[str] | None = None,
min_length: int = 20,
) -> dict[str, Any]:
report: dict[str, Any] = {
"is_valid": True,
"errors": [],
"warnings": [],
"timestamp_column": timestamp_col,
"target_columns": target_cols or [],
"missing_summary": {},
"inferred_frequency": None,
}
if df.empty:
report["errors"].append("Dataset is empty.")
report["is_valid"] = False
return report
timestamp_col = timestamp_col or detect_timestamp_column(df)
report["timestamp_column"] = timestamp_col
if not timestamp_col:
report["errors"].append("Could not detect a timestamp column.")
elif timestamp_col not in df.columns:
report["errors"].append(f"Timestamp column '{timestamp_col}' not found.")
else:
ts = pd.to_datetime(df[timestamp_col], errors="coerce")
invalid_rate = 1.0 - ts.notna().mean()
if invalid_rate > 0:
report["warnings"].append(
f"{invalid_rate:.1%} of timestamp values could not be parsed."
)
inferred = infer_frequency(df, timestamp_col)
report["inferred_frequency"] = inferred
if not inferred:
report["warnings"].append(
"Could not infer a regular frequency; forecasting still runs with best effort."
)
else:
sorted_ts = ts.dropna().sort_values().drop_duplicates()
if len(sorted_ts) >= 3:
diffs = sorted_ts.diff().dropna()
if diffs.nunique() > 1:
report["warnings"].append(
"Timestamp intervals are irregular; model accuracy may degrade."
)
if target_cols is None:
target_cols = detect_numeric_columns(df, exclude=[timestamp_col] if timestamp_col else None)
report["target_columns"] = target_cols
if not target_cols:
report["errors"].append("No numeric target columns found.")
else:
for col in target_cols:
if col not in df.columns:
report["errors"].append(f"Target column '{col}' not found.")
continue
missing = int(df[col].isna().sum())
report["missing_summary"][col] = missing
if missing > 0:
report["warnings"].append(
f"Target '{col}' contains {missing} missing values (interpolation suggested)."
)
if len(df) < min_length:
report["warnings"].append(
f"Series length ({len(df)}) is short; {min_length}+ points is recommended."
)
if report["errors"]:
report["is_valid"] = False
return report
def format_validation_report(report: dict[str, Any]) -> str:
lines: list[str] = []
status = "Valid" if report.get("is_valid") else "Invalid"
lines.append(f"Status: {status}")
timestamp_col = report.get("timestamp_column")
target_cols = report.get("target_columns", [])
freq = report.get("inferred_frequency") or "Unknown"
lines.append(f"Timestamp column: {timestamp_col or 'Not detected'}")
lines.append(f"Targets: {', '.join(target_cols) if target_cols else 'None'}")
lines.append(f"Inferred frequency: {freq}")
errors = report.get("errors", [])
warnings = report.get("warnings", [])
if errors:
lines.append("Errors:")
lines.extend(f"- {msg}" for msg in errors)
if warnings:
lines.append("Warnings:")
lines.extend(f"- {msg}" for msg in warnings)
return "\n".join(lines)
def preprocess_for_model(
df: pd.DataFrame,
model_name: str,
prediction_length: int,
timestamp_col: str | None = None,
target_cols: list[str] | None = None,
interpolate_missing: bool = True,
) -> dict[str, Any]:
timestamp_col = timestamp_col or detect_timestamp_column(df)
if not timestamp_col:
raise ValueError("Timestamp column is required for preprocessing.")
if target_cols is None or len(target_cols) == 0:
target_cols = detect_numeric_columns(df, exclude=[timestamp_col])
if not target_cols:
raise ValueError("At least one numeric target column is required.")
prepared = df.copy()
prepared[timestamp_col] = pd.to_datetime(prepared[timestamp_col], errors="coerce")
prepared = prepared.dropna(subset=[timestamp_col])
prepared = prepared.sort_values(timestamp_col)
for col in target_cols:
prepared[col] = pd.to_numeric(prepared[col], errors="coerce")
if interpolate_missing:
prepared[target_cols] = prepared[target_cols].interpolate(limit_direction="both")
prepared[target_cols] = prepared[target_cols].ffill().bfill()
if prepared[target_cols].isna().any().any():
raise ValueError("Target columns still contain NaN values after preprocessing.")
frequency = infer_frequency(prepared, timestamp_col)
context_df = prepared[[timestamp_col, *target_cols]].copy()
context_df = context_df.set_index(timestamp_col)
return {
"model_name": model_name,
"context": context_df,
"timestamp_col": timestamp_col,
"target_cols": target_cols,
"frequency": frequency,
"prediction_length": prediction_length,
"is_multivariate": len(target_cols) > 1,
}
def generate_future_index(
last_timestamp: pd.Timestamp,
prediction_length: int,
frequency: str | None,
) -> pd.DatetimeIndex:
freq = frequency or "D"
try:
return pd.date_range(start=last_timestamp, periods=prediction_length + 1, freq=freq)[1:]
except ValueError:
return pd.date_range(start=last_timestamp, periods=prediction_length + 1, freq="D")[1:]
|