idacy commited on
Commit
c789799
·
verified ·
1 Parent(s): e4b1ed6

Fix live inference slider regression

Browse files
Dockerfile CHANGED
@@ -1,8 +1,10 @@
1
  FROM python:3.11-slim
2
 
 
3
  ENV PYTHONUNBUFFERED=1
4
  ENV DCV_MODEL_RUN_DIR=data/model_runs/synthetic_v1_baseline
5
  ENV DCV_FEATURE_TABLE=data/synthetic_v1/features/window_features_all.csv
 
6
 
7
  WORKDIR /app
8
 
@@ -12,5 +14,6 @@ RUN pip install --no-cache-dir -r /tmp/requirements-api.txt
12
  COPY src ./src
13
  COPY data/model_runs/synthetic_v1_baseline ./data/model_runs/synthetic_v1_baseline
14
  COPY data/synthetic_v1/features/window_features_all.csv ./data/synthetic_v1/features/window_features_all.csv
 
15
 
16
  CMD ["sh", "-c", "uvicorn src.datacenter_verification_api.app:app --host 0.0.0.0 --port ${PORT:-7860}"]
 
1
  FROM python:3.11-slim
2
 
3
+ ARG DCV_BUILD_SHA=unknown
4
  ENV PYTHONUNBUFFERED=1
5
  ENV DCV_MODEL_RUN_DIR=data/model_runs/synthetic_v1_baseline
6
  ENV DCV_FEATURE_TABLE=data/synthetic_v1/features/window_features_all.csv
7
+ ENV DCV_BUILD_SHA=${DCV_BUILD_SHA}
8
 
9
  WORKDIR /app
10
 
 
14
  COPY src ./src
15
  COPY data/model_runs/synthetic_v1_baseline ./data/model_runs/synthetic_v1_baseline
16
  COPY data/synthetic_v1/features/window_features_all.csv ./data/synthetic_v1/features/window_features_all.csv
17
+ RUN printf '%s\n' "${DCV_BUILD_SHA}" > /app/.dcv-build-sha
18
 
19
  CMD ["sh", "-c", "uvicorn src.datacenter_verification_api.app:app --host 0.0.0.0 --port ${PORT:-7860}"]
src/datacenter_verification_api/__init__.py CHANGED
@@ -1,5 +1,76 @@
1
  """Live inference API for the datacenter verification demo."""
2
 
3
- __all__ = ["__version__"]
 
 
 
 
 
 
 
 
4
 
5
  __version__ = "0.1.0"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  """Live inference API for the datacenter verification demo."""
2
 
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import subprocess
7
+ from functools import cache
8
+ from pathlib import Path
9
+ from typing import NamedTuple
10
+
11
+ __all__ = ["__version__", "BuildInfo", "build_info"]
12
 
13
  __version__ = "0.1.0"
14
+
15
+
16
+ class BuildInfo(NamedTuple):
17
+ sha: str | None
18
+ source: str | None
19
+
20
+
21
+ def _clean_sha(value: str | None) -> str | None:
22
+ if not value:
23
+ return None
24
+ cleaned = value.strip()
25
+ if not cleaned or cleaned.lower() in {"unknown", "none", "null"}:
26
+ return None
27
+ return cleaned
28
+
29
+
30
+ def _sha_from_env() -> BuildInfo | None:
31
+ for name in (
32
+ "DCV_BUILD_SHA",
33
+ "SPACE_COMMIT_SHA",
34
+ "HF_SPACE_COMMIT_SHA",
35
+ "GITHUB_SHA",
36
+ "RENDER_GIT_COMMIT",
37
+ "COMMIT_SHA",
38
+ "SOURCE_COMMIT",
39
+ ):
40
+ sha = _clean_sha(os.getenv(name))
41
+ if sha:
42
+ return BuildInfo(sha=sha, source=f"env:{name}")
43
+ return None
44
+
45
+
46
+ def _sha_from_file() -> BuildInfo | None:
47
+ for path in (Path("/app/.dcv-build-sha"), Path.cwd() / ".dcv-build-sha"):
48
+ try:
49
+ sha = _clean_sha(path.read_text(encoding="utf-8"))
50
+ except OSError:
51
+ continue
52
+ if sha:
53
+ return BuildInfo(sha=sha, source=str(path))
54
+ return None
55
+
56
+
57
+ def _sha_from_git() -> BuildInfo | None:
58
+ try:
59
+ result = subprocess.run(
60
+ ["git", "rev-parse", "--short=12", "HEAD"],
61
+ check=True,
62
+ capture_output=True,
63
+ text=True,
64
+ timeout=1,
65
+ )
66
+ except (OSError, subprocess.SubprocessError):
67
+ return None
68
+ sha = _clean_sha(result.stdout)
69
+ if sha:
70
+ return BuildInfo(sha=sha, source="git")
71
+ return None
72
+
73
+
74
+ @cache
75
+ def build_info() -> BuildInfo:
76
+ return _sha_from_env() or _sha_from_file() or _sha_from_git() or BuildInfo(sha=None, source=None)
src/datacenter_verification_api/app.py CHANGED
@@ -9,7 +9,7 @@ from typing import AsyncIterator
9
  from fastapi import FastAPI, HTTPException
10
  from fastapi.middleware.cors import CORSMiddleware
11
 
12
- from src.datacenter_verification_api import __version__
13
  from src.datacenter_verification_api.model_service import ModelService
14
  from src.datacenter_verification_api.schemas import (
15
  BatchPredictRequest,
@@ -82,10 +82,13 @@ def require_service() -> ModelService:
82
  @app.get("/health", response_model=HealthResponse)
83
  def health() -> HealthResponse:
84
  loaded = service is not None
 
85
  return HealthResponse(
86
  status="ok" if loaded else "error",
87
  model_loaded=loaded,
88
  api_version=__version__,
 
 
89
  model_run_id=service.model_run_id if service else None,
90
  dataset_id=service.dataset_id if service else None,
91
  feature_count=len(service.feature_columns) if service else 0,
 
9
  from fastapi import FastAPI, HTTPException
10
  from fastapi.middleware.cors import CORSMiddleware
11
 
12
+ from src.datacenter_verification_api import __version__, build_info
13
  from src.datacenter_verification_api.model_service import ModelService
14
  from src.datacenter_verification_api.schemas import (
15
  BatchPredictRequest,
 
82
  @app.get("/health", response_model=HealthResponse)
83
  def health() -> HealthResponse:
84
  loaded = service is not None
85
+ build = build_info()
86
  return HealthResponse(
87
  status="ok" if loaded else "error",
88
  model_loaded=loaded,
89
  api_version=__version__,
90
+ build_sha=build.sha,
91
+ build_source=build.source,
92
  model_run_id=service.model_run_id if service else None,
93
  dataset_id=service.dataset_id if service else None,
94
  feature_count=len(service.feature_columns) if service else 0,
src/datacenter_verification_api/feature_completion.py CHANGED
@@ -20,7 +20,6 @@ FRACTION_COLUMNS = {
20
  "o3_training_sku_fraction",
21
  "o3_billing_continuity_score",
22
  "o4_gpu_util_duty_gt_70",
23
- "o4_sm_tensor_active_p95",
24
  "o4_hbm_used_fraction_p50",
25
  "o4_hbm_bandwidth_active_p95",
26
  "o4_gpu_power_fraction_p95",
@@ -282,9 +281,6 @@ def complete_features(
282
  if clamped is not None:
283
  set_if_changed(out, key, clamped, warnings, derived)
284
 
285
- if derived:
286
- warnings.append(f"derived fields updated from edited inputs: {', '.join(sorted(set(derived)))}")
287
-
288
  allocation = number_or_none(out.get("o2_max_concurrent_normalized_gpus"))
289
  capacity = number_or_none(out.get("o1_normalized_h100e_capacity"))
290
  fabric = number_or_none(out.get("o7_synchronized_fabric_footprint"))
 
20
  "o3_training_sku_fraction",
21
  "o3_billing_continuity_score",
22
  "o4_gpu_util_duty_gt_70",
 
23
  "o4_hbm_used_fraction_p50",
24
  "o4_hbm_bandwidth_active_p95",
25
  "o4_gpu_power_fraction_p95",
 
281
  if clamped is not None:
282
  set_if_changed(out, key, clamped, warnings, derived)
283
 
 
 
 
284
  allocation = number_or_none(out.get("o2_max_concurrent_normalized_gpus"))
285
  capacity = number_or_none(out.get("o1_normalized_h100e_capacity"))
286
  fabric = number_or_none(out.get("o7_synchronized_fabric_footprint"))
src/datacenter_verification_api/model_service.py CHANGED
@@ -10,7 +10,7 @@ import joblib
10
  import numpy as np
11
  import pandas as pd
12
 
13
- from src.datacenter_verification_api import __version__
14
  from src.datacenter_verification_api.feature_completion import (
15
  FeatureSchema,
16
  complete_features,
@@ -31,6 +31,11 @@ from src.datacenter_verification_modeling.common import (
31
  REPO_ROOT = Path(__file__).resolve().parents[2]
32
  DEFAULT_MODEL_RUN_DIR = Path("data/model_runs/synthetic_v1_baseline")
33
  DEFAULT_FEATURE_TABLE = Path("data/synthetic_v1/features/window_features_all.csv")
 
 
 
 
 
34
 
35
 
36
  def resolve_repo_path(value: str | Path | None, default: Path) -> Path:
@@ -147,8 +152,11 @@ class ModelService:
147
  "governance": self.metrics.get("governance", {}),
148
  "calibration": self.metrics.get("calibration", {}),
149
  }
 
150
  return MetadataResponse(
151
  api_version=__version__,
 
 
152
  model_run_id=self.model_run_id,
153
  model_run_dir=repo_relative(self.model_run_dir) or self.model_run_dir.name,
154
  feature_table=repo_relative(self.feature_table_path),
@@ -162,8 +170,9 @@ class ModelService:
162
  base_row_lookup_enabled=bool(self.feature_lookup),
163
  )
164
 
165
- def build_feature_row(self, request: PredictRequest) -> tuple[dict[str, Any], list[str]]:
166
  warnings: list[str] = []
 
167
  base_row = None
168
  if request.feature_row_id:
169
  base_row = self.feature_lookup.get(request.feature_row_id)
@@ -187,7 +196,10 @@ class ModelService:
187
  for key, value in values.items():
188
  row[key] = value
189
  changed_keys.add(key)
190
- if key not in self.feature_columns and key not in {"feature_row_id"}:
 
 
 
191
  warnings.append(f"{source_name} field is not a model feature and was kept only as metadata: {key}")
192
 
193
  for column in self.feature_columns:
@@ -208,10 +220,10 @@ class ModelService:
208
  f"no base row supplied; {null_count} of {len(self.feature_columns)} model features are null"
209
  )
210
 
211
- return row, warnings
212
 
213
  def predict(self, request: PredictRequest) -> PredictResponse:
214
- row, warnings = self.build_feature_row(request)
215
  record = {
216
  column: (np.nan if row.get(column) is None else row.get(column))
217
  for column in sorted(set(row) | set(self.feature_columns))
@@ -245,5 +257,6 @@ class ModelService:
245
  top_evidence=split_semicolon(post["top_evidence"]),
246
  critical_missing_layers=split_semicolon(post["critical_missing_layers"]),
247
  input_warnings=warnings,
 
248
  completed_features=completed_features,
249
  )
 
10
  import numpy as np
11
  import pandas as pd
12
 
13
+ from src.datacenter_verification_api import __version__, build_info
14
  from src.datacenter_verification_api.feature_completion import (
15
  FeatureSchema,
16
  complete_features,
 
31
  REPO_ROOT = Path(__file__).resolve().parents[2]
32
  DEFAULT_MODEL_RUN_DIR = Path("data/model_runs/synthetic_v1_baseline")
33
  DEFAULT_FEATURE_TABLE = Path("data/synthetic_v1/features/window_features_all.csv")
34
+ KNOWN_NON_MODEL_METADATA_FIELDS = {
35
+ "capacity_evidence_only",
36
+ "integrity_evidence_only",
37
+ "physical_evidence_only",
38
+ }
39
 
40
 
41
  def resolve_repo_path(value: str | Path | None, default: Path) -> Path:
 
152
  "governance": self.metrics.get("governance", {}),
153
  "calibration": self.metrics.get("calibration", {}),
154
  }
155
+ build = build_info()
156
  return MetadataResponse(
157
  api_version=__version__,
158
+ build_sha=build.sha,
159
+ build_source=build.source,
160
  model_run_id=self.model_run_id,
161
  model_run_dir=repo_relative(self.model_run_dir) or self.model_run_dir.name,
162
  feature_table=repo_relative(self.feature_table_path),
 
170
  base_row_lookup_enabled=bool(self.feature_lookup),
171
  )
172
 
173
+ def build_feature_row(self, request: PredictRequest) -> tuple[dict[str, Any], list[str], list[str]]:
174
  warnings: list[str] = []
175
+ debug_warnings: list[str] = []
176
  base_row = None
177
  if request.feature_row_id:
178
  base_row = self.feature_lookup.get(request.feature_row_id)
 
196
  for key, value in values.items():
197
  row[key] = value
198
  changed_keys.add(key)
199
+ if key in {"feature_row_id", *KNOWN_NON_MODEL_METADATA_FIELDS}:
200
+ if key in KNOWN_NON_MODEL_METADATA_FIELDS:
201
+ debug_warnings.append(f"{source_name} field is metadata-only and was not sent to the model: {key}")
202
+ elif key not in self.feature_columns:
203
  warnings.append(f"{source_name} field is not a model feature and was kept only as metadata: {key}")
204
 
205
  for column in self.feature_columns:
 
220
  f"no base row supplied; {null_count} of {len(self.feature_columns)} model features are null"
221
  )
222
 
223
+ return row, warnings, debug_warnings
224
 
225
  def predict(self, request: PredictRequest) -> PredictResponse:
226
+ row, warnings, debug_warnings = self.build_feature_row(request)
227
  record = {
228
  column: (np.nan if row.get(column) is None else row.get(column))
229
  for column in sorted(set(row) | set(self.feature_columns))
 
257
  top_evidence=split_semicolon(post["top_evidence"]),
258
  critical_missing_layers=split_semicolon(post["critical_missing_layers"]),
259
  input_warnings=warnings,
260
+ debug_warnings=debug_warnings,
261
  completed_features=completed_features,
262
  )
src/datacenter_verification_api/schemas.py CHANGED
@@ -32,6 +32,8 @@ class HealthResponse(BaseModel):
32
  status: str
33
  model_loaded: bool
34
  api_version: str
 
 
35
  model_run_id: str | None = None
36
  dataset_id: str | None = None
37
  feature_count: int = 0
@@ -41,6 +43,8 @@ class HealthResponse(BaseModel):
41
 
42
  class MetadataResponse(BaseModel):
43
  api_version: str
 
 
44
  model_run_id: str
45
  model_run_dir: str
46
  feature_table: str | None = None
@@ -71,6 +75,7 @@ class PredictResponse(BaseModel):
71
  top_evidence: list[str]
72
  critical_missing_layers: list[str]
73
  input_warnings: list[str] = Field(default_factory=list)
 
74
  completed_features: dict[str, Any] = Field(default_factory=dict)
75
 
76
 
 
32
  status: str
33
  model_loaded: bool
34
  api_version: str
35
+ build_sha: str | None = None
36
+ build_source: str | None = None
37
  model_run_id: str | None = None
38
  dataset_id: str | None = None
39
  feature_count: int = 0
 
43
 
44
  class MetadataResponse(BaseModel):
45
  api_version: str
46
+ build_sha: str | None = None
47
+ build_source: str | None = None
48
  model_run_id: str
49
  model_run_dir: str
50
  feature_table: str | None = None
 
75
  top_evidence: list[str]
76
  critical_missing_layers: list[str]
77
  input_warnings: list[str] = Field(default_factory=list)
78
+ debug_warnings: list[str] = Field(default_factory=list)
79
  completed_features: dict[str, Any] = Field(default_factory=dict)
80
 
81
 
src/datacenter_verification_api/smoke_test.py CHANGED
@@ -12,7 +12,13 @@ from pathlib import Path
12
 
13
  import pandas as pd
14
 
15
- from src.datacenter_verification_api.model_service import DEFAULT_MODEL_RUN_DIR, REPO_ROOT, ModelService, resolve_repo_path
 
 
 
 
 
 
16
  from src.datacenter_verification_api.schemas import PredictRequest
17
  from src.datacenter_verification_modeling.common import LABELS
18
 
@@ -84,11 +90,42 @@ def main() -> int:
84
  if not edited.completed_features:
85
  raise AssertionError("edited prediction did not return completed_features")
86
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  print(f"PASS live inference smoke test: matched {checked} exported rows from {predictions_path}")
88
  print(
89
  "PASS edited prediction schema: "
90
  f"row={edit_row_id} label={edited.predicted_label} p_large={edited.p_large_training:.6f}"
91
  )
 
 
 
 
 
92
  return 0
93
 
94
 
 
12
 
13
  import pandas as pd
14
 
15
+ from src.datacenter_verification_api.model_service import (
16
+ DEFAULT_MODEL_RUN_DIR,
17
+ KNOWN_NON_MODEL_METADATA_FIELDS,
18
+ REPO_ROOT,
19
+ ModelService,
20
+ resolve_repo_path,
21
+ )
22
  from src.datacenter_verification_api.schemas import PredictRequest
23
  from src.datacenter_verification_modeling.common import LABELS
24
 
 
90
  if not edited.completed_features:
91
  raise AssertionError("edited prediction did not return completed_features")
92
 
93
+ target_row_id = "feat_455d59646b2f3bc099ffd959"
94
+ if target_row_id not in service.feature_lookup:
95
+ target_row_id = edit_row_id
96
+ base_row = service.feature_lookup[target_row_id]
97
+ full_ui_features = {column: base_row.get(column) for column in service.feature_columns}
98
+ for column in KNOWN_NON_MODEL_METADATA_FIELDS:
99
+ full_ui_features[column] = base_row.get(column, False)
100
+ full_ui_features["o4_sm_tensor_active_p95"] = 10
101
+ tensor_edit = service.predict(
102
+ PredictRequest(
103
+ feature_row_id=target_row_id,
104
+ features=full_ui_features,
105
+ context={
106
+ "scope_type": base_row.get("scope_type") or "topology_domain",
107
+ "window_length_seconds": base_row.get("window_length_seconds") or 3600,
108
+ },
109
+ derive=True,
110
+ return_completed_features=True,
111
+ )
112
+ )
113
+ if tensor_edit.input_warnings:
114
+ raise AssertionError(f"UI-style metadata payload produced user-facing warnings: {tensor_edit.input_warnings}")
115
+ completed_tensor = tensor_edit.completed_features.get("o4_sm_tensor_active_p95")
116
+ if completed_tensor != 10:
117
+ raise AssertionError(f"tensor activity edit was not preserved; completed value is {completed_tensor!r}")
118
+
119
  print(f"PASS live inference smoke test: matched {checked} exported rows from {predictions_path}")
120
  print(
121
  "PASS edited prediction schema: "
122
  f"row={edit_row_id} label={edited.predicted_label} p_large={edited.p_large_training:.6f}"
123
  )
124
+ print(
125
+ "PASS UI payload regression: "
126
+ f"row={target_row_id} tensor={tensor_edit.completed_features.get('o4_sm_tensor_active_p95')} "
127
+ f"debug_warnings={len(tensor_edit.debug_warnings)}"
128
+ )
129
  return 0
130
 
131