minor bug fix and update quantized feature
Browse files- app.py +21 -4
- src/app_helpers/pipeline_table.py +45 -23
- src/app_helpers/queues.py +10 -4
- src/app_helpers/submissions.py +2 -0
- src/ci_dispatcher.py +57 -19
- src/populate.py +36 -4
- src/submission/submit.py +4 -0
- tests/test_auto_pipeline_results.py +10 -1
- tests/test_ci_dispatcher.py +76 -2
- tests/test_eval_queue.py +103 -0
- tests/test_submit.py +10 -3
app.py
CHANGED
|
@@ -111,7 +111,9 @@ from src.app_helpers.pipeline_table import (
|
|
| 111 |
ROW_NUMBER_COL,
|
| 112 |
add_row_numbers,
|
| 113 |
auto_pipeline_datatypes,
|
|
|
|
| 114 |
filter_auto_pipeline_table,
|
|
|
|
| 115 |
on_pipeline_dl_click,
|
| 116 |
prepare_auto_pipeline_display_df,
|
| 117 |
)
|
|
@@ -642,8 +644,15 @@ with demo:
|
|
| 642 |
f"Finished Quantizations ({len(finished_quant_queue_df)})",
|
| 643 |
open=False, elem_id="accordion-finished-quant",
|
| 644 |
) as accordion_finished_quant:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 645 |
finished_quant_table = gr.components.Dataframe(
|
| 646 |
-
value=
|
| 647 |
headers=QUANT_DISPLAY_HEADERS,
|
| 648 |
datatype=QUANT_DISPLAY_TYPES,
|
| 649 |
row_count=5,
|
|
@@ -970,6 +979,13 @@ with demo:
|
|
| 970 |
_quant_submit_evt.then(fn=refresh_queue_tables, inputs=_REFRESH_INPUTS, outputs=_REFRESH_OUTPUTS)
|
| 971 |
_eval_submit_evt.then(fn=refresh_queue_tables, inputs=_REFRESH_INPUTS, outputs=_REFRESH_OUTPUTS)
|
| 972 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 973 |
|
| 974 |
# Checkbox filters β instantly re-filter the queue tables
|
| 975 |
my_submissions_quant_cb.change(
|
|
@@ -1079,8 +1095,8 @@ with demo:
|
|
| 1079 |
}""",
|
| 1080 |
)
|
| 1081 |
|
| 1082 |
-
# ββ Live queue refresh (every
|
| 1083 |
-
_queue_timer = gr.Timer(value=
|
| 1084 |
_queue_timer.tick(
|
| 1085 |
fn=refresh_queue_tables,
|
| 1086 |
inputs=_REFRESH_INPUTS,
|
|
@@ -1097,7 +1113,8 @@ with demo:
|
|
| 1097 |
auto_pipeline_size_slider,
|
| 1098 |
]
|
| 1099 |
_pipeline_refresh_outputs = [hidden_auto_pipeline_table, auto_pipeline_table]
|
| 1100 |
-
|
|
|
|
| 1101 |
fn=refresh_pipeline_leaderboard,
|
| 1102 |
inputs=_pipeline_refresh_inputs,
|
| 1103 |
outputs=_pipeline_refresh_outputs,
|
|
|
|
| 111 |
ROW_NUMBER_COL,
|
| 112 |
add_row_numbers,
|
| 113 |
auto_pipeline_datatypes,
|
| 114 |
+
embed_dl_icon_in_hf_model_cell,
|
| 115 |
filter_auto_pipeline_table,
|
| 116 |
+
on_queue_dl_click,
|
| 117 |
on_pipeline_dl_click,
|
| 118 |
prepare_auto_pipeline_display_df,
|
| 119 |
)
|
|
|
|
| 644 |
f"Finished Quantizations ({len(finished_quant_queue_df)})",
|
| 645 |
open=False, elem_id="accordion-finished-quant",
|
| 646 |
) as accordion_finished_quant:
|
| 647 |
+
_finished_quant_display_df = (
|
| 648 |
+
finished_quant_queue_df[QUANT_DISPLAY_COLS].copy()
|
| 649 |
+
if not finished_quant_queue_df.empty
|
| 650 |
+
else finished_quant_queue_df.copy()
|
| 651 |
+
)
|
| 652 |
+
if "model" in _finished_quant_display_df.columns:
|
| 653 |
+
_finished_quant_display_df["model"] = _finished_quant_display_df["model"].map(embed_dl_icon_in_hf_model_cell)
|
| 654 |
finished_quant_table = gr.components.Dataframe(
|
| 655 |
+
value=_finished_quant_display_df,
|
| 656 |
headers=QUANT_DISPLAY_HEADERS,
|
| 657 |
datatype=QUANT_DISPLAY_TYPES,
|
| 658 |
row_count=5,
|
|
|
|
| 979 |
_quant_submit_evt.then(fn=refresh_queue_tables, inputs=_REFRESH_INPUTS, outputs=_REFRESH_OUTPUTS)
|
| 980 |
_eval_submit_evt.then(fn=refresh_queue_tables, inputs=_REFRESH_INPUTS, outputs=_REFRESH_OUTPUTS)
|
| 981 |
|
| 982 |
+
finished_quant_table.select(
|
| 983 |
+
on_queue_dl_click,
|
| 984 |
+
inputs=[],
|
| 985 |
+
outputs=[pipeline_dl_trigger],
|
| 986 |
+
queue=True,
|
| 987 |
+
)
|
| 988 |
+
|
| 989 |
|
| 990 |
# Checkbox filters β instantly re-filter the queue tables
|
| 991 |
my_submissions_quant_cb.change(
|
|
|
|
| 1095 |
}""",
|
| 1096 |
)
|
| 1097 |
|
| 1098 |
+
# ββ Live queue refresh (every 5 s so dispatcher status changes surface promptly) βββββββββ
|
| 1099 |
+
_queue_timer = gr.Timer(value=5)
|
| 1100 |
_queue_timer.tick(
|
| 1101 |
fn=refresh_queue_tables,
|
| 1102 |
inputs=_REFRESH_INPUTS,
|
|
|
|
| 1113 |
auto_pipeline_size_slider,
|
| 1114 |
]
|
| 1115 |
_pipeline_refresh_outputs = [hidden_auto_pipeline_table, auto_pipeline_table]
|
| 1116 |
+
_pipeline_timer = gr.Timer(value=60)
|
| 1117 |
+
_pipeline_timer.tick(
|
| 1118 |
fn=refresh_pipeline_leaderboard,
|
| 1119 |
inputs=_pipeline_refresh_inputs,
|
| 1120 |
outputs=_pipeline_refresh_outputs,
|
src/app_helpers/pipeline_table.py
CHANGED
|
@@ -68,20 +68,31 @@ def prepare_auto_pipeline_display_df(df: pd.DataFrame) -> pd.DataFrame:
|
|
| 68 |
return display_df.fillna("")
|
| 69 |
|
| 70 |
|
| 71 |
-
def
|
| 72 |
-
"""
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
return
|
| 77 |
-
|
| 78 |
-
|
|
|
|
|
|
|
| 79 |
|
| 80 |
hf_match = re.search(r'href=["\']https://huggingface\.co/([^"\'\/][^"\']*)["\' \\s]', cell)
|
| 81 |
if hf_match:
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
if not artifact or "/" not in artifact:
|
| 86 |
return cell
|
| 87 |
|
|
@@ -92,8 +103,17 @@ def _embed_dl_icon_in_quantized(row):
|
|
| 92 |
f'color:#2563eb;font-size:11px;font-weight:700;line-height:16px;vertical-align:middle;'
|
| 93 |
f'padding-bottom:1px;">β</span>'
|
| 94 |
)
|
| 95 |
-
new_cell = re.sub(r"</a>", icon_html + "</a>", cell, count=1)
|
| 96 |
-
return new_cell if new_cell != cell else cell + icon_html
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
|
| 98 |
|
| 99 |
def _search_auto_pipeline_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
|
|
@@ -203,16 +223,9 @@ def on_pipeline_dl_click(select_data: gr.SelectData, hidden_df: pd.DataFrame) ->
|
|
| 203 |
if "dl-cell-link" not in cell_value and "#hf/" not in cell_value:
|
| 204 |
return ""
|
| 205 |
|
| 206 |
-
|
| 207 |
-
if
|
| 208 |
-
|
| 209 |
-
if "/" in artifact:
|
| 210 |
-
return artifact
|
| 211 |
-
title_match = re.search(r'\btitle=["\']([^"\']+/[^"\']+)["\']', cell_value)
|
| 212 |
-
if title_match:
|
| 213 |
-
artifact = title_match.group(1).strip()
|
| 214 |
-
if "/" in artifact:
|
| 215 |
-
return artifact
|
| 216 |
|
| 217 |
# Last-resort fallback: look up by row index (may be wrong when sorted).
|
| 218 |
row_idx = select_data.index[0]
|
|
@@ -229,3 +242,12 @@ def on_pipeline_dl_click(select_data: gr.SelectData, hidden_df: pd.DataFrame) ->
|
|
| 229 |
except Exception:
|
| 230 |
pass
|
| 231 |
return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
return display_df.fillna("")
|
| 69 |
|
| 70 |
|
| 71 |
+
def _extract_hf_repo(value: str) -> str:
|
| 72 |
+
"""Return an ``org/repo`` id from a Hugging Face link or markdown/html cell."""
|
| 73 |
+
cell = str(value or "")
|
| 74 |
+
hf_hash = re.search(r'#hf/([^\s"\'>/]+(?:/[^\s"\'>/]+)+)', cell)
|
| 75 |
+
if hf_hash:
|
| 76 |
+
return hf_hash.group(1).rstrip("/")
|
| 77 |
+
|
| 78 |
+
title_match = re.search(r'\btitle=["\']([^"\']+/[^"\']+)["\']', cell)
|
| 79 |
+
if title_match:
|
| 80 |
+
return title_match.group(1).strip().rstrip("/")
|
| 81 |
|
| 82 |
hf_match = re.search(r'href=["\']https://huggingface\.co/([^"\'\/][^"\']*)["\' \\s]', cell)
|
| 83 |
if hf_match:
|
| 84 |
+
return hf_match.group(1).rstrip("/")
|
| 85 |
+
|
| 86 |
+
plain = re.sub(r"<[^>]+>", "", cell).strip()
|
| 87 |
+
return plain.rstrip("/") if "/" in plain else ""
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def embed_dl_icon_in_hf_model_cell(cell: str) -> str:
|
| 91 |
+
"""Insert a download-icon span into a Hugging Face model cell."""
|
| 92 |
+
if not cell or not str(cell).strip() or "dl-cell-link" in str(cell):
|
| 93 |
+
return cell
|
| 94 |
+
|
| 95 |
+
artifact = _extract_hf_repo(str(cell))
|
| 96 |
if not artifact or "/" not in artifact:
|
| 97 |
return cell
|
| 98 |
|
|
|
|
| 103 |
f'color:#2563eb;font-size:11px;font-weight:700;line-height:16px;vertical-align:middle;'
|
| 104 |
f'padding-bottom:1px;">β</span>'
|
| 105 |
)
|
| 106 |
+
new_cell = re.sub(r"</a>", icon_html + "</a>", str(cell), count=1)
|
| 107 |
+
return new_cell if new_cell != str(cell) else str(cell) + icon_html
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def _embed_dl_icon_in_quantized(row):
|
| 111 |
+
"""Insert a download-icon span just before the ``</a>`` in the model link."""
|
| 112 |
+
try:
|
| 113 |
+
cell = str(row["Quantized Model"])
|
| 114 |
+
except Exception:
|
| 115 |
+
return row["Quantized Model"]
|
| 116 |
+
return embed_dl_icon_in_hf_model_cell(cell)
|
| 117 |
|
| 118 |
|
| 119 |
def _search_auto_pipeline_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
|
|
|
|
| 223 |
if "dl-cell-link" not in cell_value and "#hf/" not in cell_value:
|
| 224 |
return ""
|
| 225 |
|
| 226 |
+
artifact = _extract_hf_repo(cell_value)
|
| 227 |
+
if artifact and "/" in artifact:
|
| 228 |
+
return artifact
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 229 |
|
| 230 |
# Last-resort fallback: look up by row index (may be wrong when sorted).
|
| 231 |
row_idx = select_data.index[0]
|
|
|
|
| 242 |
except Exception:
|
| 243 |
pass
|
| 244 |
return ""
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def on_queue_dl_click(select_data: gr.SelectData) -> str:
|
| 248 |
+
"""Return the repo id from a queue-table download icon click."""
|
| 249 |
+
cell_value = str(select_data.value or "")
|
| 250 |
+
if "dl-cell-link" not in cell_value and "#hf/" not in cell_value:
|
| 251 |
+
return ""
|
| 252 |
+
artifact = _extract_hf_repo(cell_value)
|
| 253 |
+
return artifact if artifact and "/" in artifact else ""
|
src/app_helpers/queues.py
CHANGED
|
@@ -7,7 +7,10 @@ import re
|
|
| 7 |
import gradio as gr
|
| 8 |
import pandas as pd
|
| 9 |
|
| 10 |
-
from src.app_helpers.pipeline_table import
|
|
|
|
|
|
|
|
|
|
| 11 |
from src.display.utils import (
|
| 12 |
EVAL_COLS,
|
| 13 |
EVAL_DISPLAY_COLS,
|
|
@@ -46,9 +49,12 @@ def refresh_queue_tables(
|
|
| 46 |
return df[df["submitted_by"] == username].reset_index(drop=True)
|
| 47 |
return df
|
| 48 |
|
| 49 |
-
def _quant_display(df):
|
| 50 |
cols = [c for c in QUANT_DISPLAY_COLS if c in df.columns]
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
def _eval_display(df):
|
| 54 |
cols = [c for c in EVAL_DISPLAY_COLS if c in df.columns]
|
|
@@ -76,7 +82,7 @@ def refresh_queue_tables(
|
|
| 76 |
fail_e = _search_queue_df(_filt(new_failed_eval, show_mine_eval), query_eval)
|
| 77 |
|
| 78 |
return (
|
| 79 |
-
_quant_display(fin_q),
|
| 80 |
_quant_display(run_q),
|
| 81 |
_quant_display(pen_q),
|
| 82 |
_quant_display(fail_q),
|
|
|
|
| 7 |
import gradio as gr
|
| 8 |
import pandas as pd
|
| 9 |
|
| 10 |
+
from src.app_helpers.pipeline_table import (
|
| 11 |
+
embed_dl_icon_in_hf_model_cell,
|
| 12 |
+
filter_auto_pipeline_table,
|
| 13 |
+
)
|
| 14 |
from src.display.utils import (
|
| 15 |
EVAL_COLS,
|
| 16 |
EVAL_DISPLAY_COLS,
|
|
|
|
| 49 |
return df[df["submitted_by"] == username].reset_index(drop=True)
|
| 50 |
return df
|
| 51 |
|
| 52 |
+
def _quant_display(df, with_download_icon=False):
|
| 53 |
cols = [c for c in QUANT_DISPLAY_COLS if c in df.columns]
|
| 54 |
+
out = df[cols].copy() if cols else df.copy()
|
| 55 |
+
if with_download_icon and "model" in out.columns:
|
| 56 |
+
out["model"] = out["model"].map(embed_dl_icon_in_hf_model_cell)
|
| 57 |
+
return out
|
| 58 |
|
| 59 |
def _eval_display(df):
|
| 60 |
cols = [c for c in EVAL_DISPLAY_COLS if c in df.columns]
|
|
|
|
| 82 |
fail_e = _search_queue_df(_filt(new_failed_eval, show_mine_eval), query_eval)
|
| 83 |
|
| 84 |
return (
|
| 85 |
+
_quant_display(fin_q, with_download_icon=True),
|
| 86 |
_quant_display(run_q),
|
| 87 |
_quant_display(pen_q),
|
| 88 |
_quant_display(fail_q),
|
src/app_helpers/submissions.py
CHANGED
|
@@ -211,6 +211,7 @@ def submit_model(
|
|
| 211 |
hardware_override=hw_override,
|
| 212 |
gpu_count_override=count_override,
|
| 213 |
submitted_by=username,
|
|
|
|
| 214 |
user_token=oauth_token.token if oauth_token else None,
|
| 215 |
)
|
| 216 |
)
|
|
@@ -278,6 +279,7 @@ def submit_quant(
|
|
| 278 |
hardware_override=hw_override,
|
| 279 |
gpu_count_override=count_override,
|
| 280 |
submitted_by=username,
|
|
|
|
| 281 |
user_token=oauth_token.token if oauth_token else None,
|
| 282 |
)
|
| 283 |
)
|
|
|
|
| 211 |
hardware_override=hw_override,
|
| 212 |
gpu_count_override=count_override,
|
| 213 |
submitted_by=username,
|
| 214 |
+
submitted_orgs=orgs,
|
| 215 |
user_token=oauth_token.token if oauth_token else None,
|
| 216 |
)
|
| 217 |
)
|
|
|
|
| 279 |
hardware_override=hw_override,
|
| 280 |
gpu_count_override=count_override,
|
| 281 |
submitted_by=username,
|
| 282 |
+
submitted_orgs=orgs,
|
| 283 |
user_token=oauth_token.token if oauth_token else None,
|
| 284 |
)
|
| 285 |
)
|
src/ci_dispatcher.py
CHANGED
|
@@ -13,15 +13,24 @@ Concurrency control:
|
|
| 13 |
``max_instances=1`` + ``coalesce=True`` prevent overlapping scans.
|
| 14 |
|
| 15 |
Active status reconciliation:
|
| 16 |
-
|
| 17 |
-
``ci_run_id``
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
Time-based guards cover ALL failure modes:
|
| 19 |
|
| 20 |
-
1. Azure ``completed+failed/canceled`` β
|
| 21 |
-
2. Azure ``completed+succeeded`` but no write-back after grace period β
|
| 22 |
-
3. Azure ``inProgress`` beyond max runtime β
|
| 23 |
-
4. Azure API unreachable beyond fallback timeout β
|
| 24 |
-
5.
|
| 25 |
"""
|
| 26 |
|
| 27 |
import base64
|
|
@@ -104,7 +113,8 @@ class CIDispatcher:
|
|
| 104 |
if not self._enable_retry:
|
| 105 |
logger.info(
|
| 106 |
"CIDispatcher: automatic retry is DISABLED. "
|
| 107 |
-
"
|
|
|
|
| 108 |
)
|
| 109 |
|
| 110 |
# ββ public entry point (called by APScheduler) βββββββββββββββββββββββ
|
|
@@ -124,11 +134,12 @@ class CIDispatcher:
|
|
| 124 |
logger.error("[dispatcher] Repo sync failed, skipping cycle", exc_info=True)
|
| 125 |
return
|
| 126 |
|
| 127 |
-
# 2) Reconcile active entries against Azure real status
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
|
|
|
| 132 |
|
| 133 |
# 3) Collect status summary for logging
|
| 134 |
running_models, triggered_models, pending_models = self._collect_status_summary()
|
|
@@ -402,7 +413,7 @@ class CIDispatcher:
|
|
| 402 |
# ββ No ci_run_id (e.g. Triggered but API call never returned) ββ
|
| 403 |
if not run_id:
|
| 404 |
if triggered_at and self._hours_since(triggered_at) > _API_UNREACHABLE_HOURS:
|
| 405 |
-
new_status = self.
|
| 406 |
self._reset_active_fields(data, new_status)
|
| 407 |
updates.append((fpath, data))
|
| 408 |
logger.warning(
|
|
@@ -417,7 +428,7 @@ class CIDispatcher:
|
|
| 417 |
if azure_state is None:
|
| 418 |
# API unreachable β apply fallback timeout
|
| 419 |
if triggered_at and self._hours_since(triggered_at) > _API_UNREACHABLE_HOURS:
|
| 420 |
-
new_status = self.
|
| 421 |
self._reset_active_fields(data, new_status)
|
| 422 |
updates.append((fpath, data))
|
| 423 |
logger.warning(
|
|
@@ -436,7 +447,7 @@ class CIDispatcher:
|
|
| 436 |
# writes back Finished. Give it a grace period.
|
| 437 |
if triggered_at and self._hours_since(triggered_at) > _SUCCEEDED_GRACE_HOURS:
|
| 438 |
# Grace period expired β write-back likely failed
|
| 439 |
-
new_status = self.
|
| 440 |
self._reset_active_fields(data, new_status)
|
| 441 |
updates.append((fpath, data))
|
| 442 |
logger.warning(
|
|
@@ -449,8 +460,9 @@ class CIDispatcher:
|
|
| 449 |
data["status"] = "Running"
|
| 450 |
updates.append((fpath, data))
|
| 451 |
else:
|
| 452 |
-
# failed or canceled β retry
|
| 453 |
-
|
|
|
|
| 454 |
self._reset_active_fields(data, new_status)
|
| 455 |
updates.append((fpath, data))
|
| 456 |
logger.info(
|
|
@@ -462,7 +474,7 @@ class CIDispatcher:
|
|
| 462 |
# ββ Azure: inProgress / canceling ββ
|
| 463 |
# Check absolute max runtime
|
| 464 |
if triggered_at and self._hours_since(triggered_at) > _MAX_ACTIVE_HOURS:
|
| 465 |
-
new_status = self.
|
| 466 |
self._reset_active_fields(data, new_status)
|
| 467 |
updates.append((fpath, data))
|
| 468 |
logger.warning(
|
|
@@ -573,6 +585,12 @@ class CIDispatcher:
|
|
| 573 |
data["last_failed_time"] = (
|
| 574 |
datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 575 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 576 |
|
| 577 |
@staticmethod
|
| 578 |
def _apply_retry_limit(data: dict, candidate_status: str, model: str) -> str:
|
|
@@ -600,3 +618,23 @@ class CIDispatcher:
|
|
| 600 |
model, retry_count, _MAX_RETRIES,
|
| 601 |
)
|
| 602 |
return "Pending"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
``max_instances=1`` + ``coalesce=True`` prevent overlapping scans.
|
| 14 |
|
| 15 |
Active status reconciliation:
|
| 16 |
+
Every scan cycle queries the Azure DevOps Runs API for entries that have a
|
| 17 |
+
``ci_run_id`` (independent of ``enable_retry`` so the UI always reflects the
|
| 18 |
+
real pipeline state). The real pipeline state is mapped back to local
|
| 19 |
+
status. How a *failure* resolves depends on ``enable_retry``:
|
| 20 |
+
|
| 21 |
+
* ``enable_retry=True`` β failures reset to ``Pending`` (re-dispatched),
|
| 22 |
+
up to ``_MAX_RETRIES`` then ``Failed``.
|
| 23 |
+
* ``enable_retry=False`` β failures map to a terminal failed status
|
| 24 |
+
(``Quant Failed`` / ``Eval Failed``) shown immediately in the UI; the
|
| 25 |
+
entry is not re-dispatched.
|
| 26 |
+
|
| 27 |
Time-based guards cover ALL failure modes:
|
| 28 |
|
| 29 |
+
1. Azure ``completed+failed/canceled`` β resolve failure (immediate)
|
| 30 |
+
2. Azure ``completed+succeeded`` but no write-back after grace period β resolve failure
|
| 31 |
+
3. Azure ``inProgress`` beyond max runtime β resolve failure (pipeline hung)
|
| 32 |
+
4. Azure API unreachable beyond fallback timeout β resolve failure
|
| 33 |
+
5. With retry enabled, all resets go through ``_MAX_RETRIES`` limit β ``Failed``
|
| 34 |
"""
|
| 35 |
|
| 36 |
import base64
|
|
|
|
| 113 |
if not self._enable_retry:
|
| 114 |
logger.info(
|
| 115 |
"CIDispatcher: automatic retry is DISABLED. "
|
| 116 |
+
"Azure failures are reconciled to a terminal failed status "
|
| 117 |
+
"(Quant Failed / Eval Failed) instead of being re-dispatched."
|
| 118 |
)
|
| 119 |
|
| 120 |
# ββ public entry point (called by APScheduler) βββββββββββββββββββββββ
|
|
|
|
| 134 |
logger.error("[dispatcher] Repo sync failed, skipping cycle", exc_info=True)
|
| 135 |
return
|
| 136 |
|
| 137 |
+
# 2) Reconcile active entries against Azure real status.
|
| 138 |
+
# Always runs (even with retry disabled) so the UI reflects the real
|
| 139 |
+
# Azure outcome. ``enable_retry`` only controls what a *failure*
|
| 140 |
+
# resolves to: a Pending re-dispatch (retry) vs a terminal failed
|
| 141 |
+
# status (no retry). See ``_resolve_failure``.
|
| 142 |
+
self._reconcile_active_runs()
|
| 143 |
|
| 144 |
# 3) Collect status summary for logging
|
| 145 |
running_models, triggered_models, pending_models = self._collect_status_summary()
|
|
|
|
| 413 |
# ββ No ci_run_id (e.g. Triggered but API call never returned) ββ
|
| 414 |
if not run_id:
|
| 415 |
if triggered_at and self._hours_since(triggered_at) > _API_UNREACHABLE_HOURS:
|
| 416 |
+
new_status = self._resolve_failure(data, model)
|
| 417 |
self._reset_active_fields(data, new_status)
|
| 418 |
updates.append((fpath, data))
|
| 419 |
logger.warning(
|
|
|
|
| 428 |
if azure_state is None:
|
| 429 |
# API unreachable β apply fallback timeout
|
| 430 |
if triggered_at and self._hours_since(triggered_at) > _API_UNREACHABLE_HOURS:
|
| 431 |
+
new_status = self._resolve_failure(data, model)
|
| 432 |
self._reset_active_fields(data, new_status)
|
| 433 |
updates.append((fpath, data))
|
| 434 |
logger.warning(
|
|
|
|
| 447 |
# writes back Finished. Give it a grace period.
|
| 448 |
if triggered_at and self._hours_since(triggered_at) > _SUCCEEDED_GRACE_HOURS:
|
| 449 |
# Grace period expired β write-back likely failed
|
| 450 |
+
new_status = self._resolve_failure(data, model)
|
| 451 |
self._reset_active_fields(data, new_status)
|
| 452 |
updates.append((fpath, data))
|
| 453 |
logger.warning(
|
|
|
|
| 460 |
data["status"] = "Running"
|
| 461 |
updates.append((fpath, data))
|
| 462 |
else:
|
| 463 |
+
# failed or canceled β resolve to retry (Pending) or a
|
| 464 |
+
# terminal failed status depending on enable_retry
|
| 465 |
+
new_status = self._resolve_failure(data, model)
|
| 466 |
self._reset_active_fields(data, new_status)
|
| 467 |
updates.append((fpath, data))
|
| 468 |
logger.info(
|
|
|
|
| 474 |
# ββ Azure: inProgress / canceling ββ
|
| 475 |
# Check absolute max runtime
|
| 476 |
if triggered_at and self._hours_since(triggered_at) > _MAX_ACTIVE_HOURS:
|
| 477 |
+
new_status = self._resolve_failure(data, model)
|
| 478 |
self._reset_active_fields(data, new_status)
|
| 479 |
updates.append((fpath, data))
|
| 480 |
logger.warning(
|
|
|
|
| 585 |
data["last_failed_time"] = (
|
| 586 |
datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 587 |
)
|
| 588 |
+
elif new_status in ("Quant Failed", "Eval Failed"):
|
| 589 |
+
# Terminal failure surfaced from Azure (retry disabled). Keep
|
| 590 |
+
# ci_run_id so the Azure run stays inspectable; just stamp the time.
|
| 591 |
+
data["last_failed_time"] = (
|
| 592 |
+
datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 593 |
+
)
|
| 594 |
|
| 595 |
@staticmethod
|
| 596 |
def _apply_retry_limit(data: dict, candidate_status: str, model: str) -> str:
|
|
|
|
| 618 |
model, retry_count, _MAX_RETRIES,
|
| 619 |
)
|
| 620 |
return "Pending"
|
| 621 |
+
|
| 622 |
+
def _resolve_failure(self, data: dict, model: str) -> str:
|
| 623 |
+
"""Decide what an active run that cannot continue should become.
|
| 624 |
+
|
| 625 |
+
* ``enable_retry=True`` β reset to ``Pending`` (re-dispatched next
|
| 626 |
+
cycle) until ``_MAX_RETRIES`` is exhausted, then ``Failed``.
|
| 627 |
+
* ``enable_retry=False`` β map to a terminal failed status
|
| 628 |
+
(``Quant Failed`` / ``Eval Failed``) so the UI reflects the failure
|
| 629 |
+
immediately and the entry is NOT re-dispatched.
|
| 630 |
+
"""
|
| 631 |
+
if self._enable_retry:
|
| 632 |
+
return self._apply_retry_limit(data, "Pending", model)
|
| 633 |
+
return self._terminal_failed_status(data)
|
| 634 |
+
|
| 635 |
+
@staticmethod
|
| 636 |
+
def _terminal_failed_status(data: dict) -> str:
|
| 637 |
+
"""Terminal failed status for an entry, keyed off its pipeline script."""
|
| 638 |
+
if data.get("script") == "auto_eval":
|
| 639 |
+
return "Eval Failed"
|
| 640 |
+
return "Quant Failed"
|
src/populate.py
CHANGED
|
@@ -185,7 +185,15 @@ def _infer_quant_status(entry: dict, result_index: dict) -> str:
|
|
| 185 |
agg = entry.get("_matched_aggregate")
|
| 186 |
if agg is None:
|
| 187 |
return original_status
|
| 188 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
|
| 190 |
|
| 191 |
def _infer_eval_status(entry: dict, result_index: dict) -> str:
|
|
@@ -336,9 +344,16 @@ def _load_queue_entries(save_path: str, request_type: str = None) -> list[dict]:
|
|
| 336 |
|
| 337 |
|
| 338 |
def _split_by_status(all_evals: list[dict]) -> tuple[list, list, list, list]:
|
| 339 |
-
"""Split entries into (pending, running, finished, failed) lists by status.
|
| 340 |
-
|
| 341 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 342 |
finished = [e for e in all_evals
|
| 343 |
if e.get("status", "").startswith("Finished")
|
| 344 |
or e.get("status") == "PENDING_NEW_EVAL"]
|
|
@@ -351,6 +366,21 @@ def _split_by_status(all_evals: list[dict]) -> tuple[list, list, list, list]:
|
|
| 351 |
return pending, running, finished, failed
|
| 352 |
|
| 353 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 354 |
def _build_queue_dfs(pending: list, running: list, finished: list, failed: list,
|
| 355 |
cols: list) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
| 356 |
"""Build DataFrames from split lists, keeping only the requested *cols*."""
|
|
@@ -438,6 +468,8 @@ def get_evaluation_queue_df(save_path: str, cols: list,
|
|
| 438 |
entry["status"] = _infer_eval_status(entry, result_index)
|
| 439 |
|
| 440 |
pending, running, finished, failed = _split_by_status(all_evals)
|
|
|
|
|
|
|
| 441 |
|
| 442 |
# Inject ETA for pending entries
|
| 443 |
_inject_eta(pending, running, concurrency=2)
|
|
|
|
| 185 |
agg = entry.get("_matched_aggregate")
|
| 186 |
if agg is None:
|
| 187 |
return original_status
|
| 188 |
+
derived = _derive_status_from_aggregate(agg)
|
| 189 |
+
# The entry is still actively running on Azure (Running/Triggered/Waiting).
|
| 190 |
+
# A *partial* aggregate (quant done but eval not finished) must NOT pull it
|
| 191 |
+
# out of the Running queue β only a terminal outcome (Finished / a failure)
|
| 192 |
+
# may override a live active status. Otherwise the dispatcher log shows
|
| 193 |
+
# "Running" while the UI wrongly lists it as pending/finished.
|
| 194 |
+
if original_status in ("Running", "Triggered", "Waiting") and derived in ("Quantized", "Partial"):
|
| 195 |
+
return original_status
|
| 196 |
+
return derived
|
| 197 |
|
| 198 |
|
| 199 |
def _infer_eval_status(entry: dict, result_index: dict) -> str:
|
|
|
|
| 344 |
|
| 345 |
|
| 346 |
def _split_by_status(all_evals: list[dict]) -> tuple[list, list, list, list]:
|
| 347 |
+
"""Split entries into (pending, running, finished, failed) lists by status.
|
| 348 |
+
|
| 349 |
+
``Waiting`` is an *active* status (the CI run is queued/executing on Azure)
|
| 350 |
+
and is treated as Running everywhere else in the system (CIDispatcher
|
| 351 |
+
``_collect_status_summary``, ``queue_eta``, retry logic). It must live in
|
| 352 |
+
the running bucket so the UI matches the dispatcher log instead of showing
|
| 353 |
+
a model as pending while the log already reports it as Running.
|
| 354 |
+
"""
|
| 355 |
+
pending = [e for e in all_evals if e.get("status") in ("Pending", "Rerun", "Quantized")]
|
| 356 |
+
running = [e for e in all_evals if e.get("status") in ("Running", "Triggered", "Waiting")]
|
| 357 |
finished = [e for e in all_evals
|
| 358 |
if e.get("status", "").startswith("Finished")
|
| 359 |
or e.get("status") == "PENDING_NEW_EVAL"]
|
|
|
|
| 366 |
return pending, running, finished, failed
|
| 367 |
|
| 368 |
|
| 369 |
+
def _drop_resolved_quant_failures(failed: list[dict], all_entries: list[dict]) -> list[dict]:
|
| 370 |
+
"""Remove failed quant entries superseded by a later submission."""
|
| 371 |
+
latest_submission_by_key: dict[tuple[str, str], str] = {}
|
| 372 |
+
for entry in all_entries:
|
| 373 |
+
key = _quant_match_key(entry)
|
| 374 |
+
submitted_time = entry.get("submitted_time", "")
|
| 375 |
+
if submitted_time >= latest_submission_by_key.get(key, ""):
|
| 376 |
+
latest_submission_by_key[key] = submitted_time
|
| 377 |
+
|
| 378 |
+
return [
|
| 379 |
+
entry for entry in failed
|
| 380 |
+
if latest_submission_by_key.get(_quant_match_key(entry), "") <= entry.get("submitted_time", "")
|
| 381 |
+
]
|
| 382 |
+
|
| 383 |
+
|
| 384 |
def _build_queue_dfs(pending: list, running: list, finished: list, failed: list,
|
| 385 |
cols: list) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
| 386 |
"""Build DataFrames from split lists, keeping only the requested *cols*."""
|
|
|
|
| 468 |
entry["status"] = _infer_eval_status(entry, result_index)
|
| 469 |
|
| 470 |
pending, running, finished, failed = _split_by_status(all_evals)
|
| 471 |
+
if request_type == "quant":
|
| 472 |
+
failed = _drop_resolved_quant_failures(failed, all_evals)
|
| 473 |
|
| 474 |
# Inject ETA for pending entries
|
| 475 |
_inject_eta(pending, running, concurrency=2)
|
src/submission/submit.py
CHANGED
|
@@ -381,6 +381,7 @@ def add_new_eval(
|
|
| 381 |
hardware_override: str | None = None,
|
| 382 |
gpu_count_override: int | None = None,
|
| 383 |
submitted_by: str = "",
|
|
|
|
| 384 |
user_token: str | None = None,
|
| 385 |
):
|
| 386 |
"""Submit an already-quantized model for evaluation (generator with progress).
|
|
@@ -620,6 +621,7 @@ def add_new_eval(
|
|
| 620 |
"status": "Pending",
|
| 621 |
"submitted_time": current_time,
|
| 622 |
"submitted_by": submitted_by,
|
|
|
|
| 623 |
"model_type": "quantization",
|
| 624 |
"job_id": -1,
|
| 625 |
"job_start_time": None,
|
|
@@ -684,6 +686,7 @@ def add_new_quant(
|
|
| 684 |
hardware_override: str | None = None,
|
| 685 |
gpu_count_override: int | None = None,
|
| 686 |
submitted_by: str = "",
|
|
|
|
| 687 |
user_token: str | None = None,
|
| 688 |
):
|
| 689 |
"""Submit an FP model for quantization + evaluation (generator with progress).
|
|
@@ -907,6 +910,7 @@ def add_new_quant(
|
|
| 907 |
"status": "Pending",
|
| 908 |
"submitted_time": current_time,
|
| 909 |
"submitted_by": submitted_by,
|
|
|
|
| 910 |
"model_type": "quantization",
|
| 911 |
"job_id": -1,
|
| 912 |
"job_start_time": None,
|
|
|
|
| 381 |
hardware_override: str | None = None,
|
| 382 |
gpu_count_override: int | None = None,
|
| 383 |
submitted_by: str = "",
|
| 384 |
+
submitted_orgs: list[str] | None = None,
|
| 385 |
user_token: str | None = None,
|
| 386 |
):
|
| 387 |
"""Submit an already-quantized model for evaluation (generator with progress).
|
|
|
|
| 621 |
"status": "Pending",
|
| 622 |
"submitted_time": current_time,
|
| 623 |
"submitted_by": submitted_by,
|
| 624 |
+
"submitted_orgs": submitted_orgs or [],
|
| 625 |
"model_type": "quantization",
|
| 626 |
"job_id": -1,
|
| 627 |
"job_start_time": None,
|
|
|
|
| 686 |
hardware_override: str | None = None,
|
| 687 |
gpu_count_override: int | None = None,
|
| 688 |
submitted_by: str = "",
|
| 689 |
+
submitted_orgs: list[str] | None = None,
|
| 690 |
user_token: str | None = None,
|
| 691 |
):
|
| 692 |
"""Submit an FP model for quantization + evaluation (generator with progress).
|
|
|
|
| 910 |
"status": "Pending",
|
| 911 |
"submitted_time": current_time,
|
| 912 |
"submitted_by": submitted_by,
|
| 913 |
+
"submitted_orgs": submitted_orgs or [],
|
| 914 |
"model_type": "quantization",
|
| 915 |
"job_id": -1,
|
| 916 |
"job_start_time": None,
|
tests/test_auto_pipeline_results.py
CHANGED
|
@@ -8,7 +8,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
| 8 |
|
| 9 |
from src.leaderboard.read_auto_pipeline_results import get_auto_pipeline_results_df
|
| 10 |
from src.leaderboard.read_auto_pipeline_results import AUTO_PIPELINE_SEARCH_COL, AUTO_PIPELINE_SORT_COL
|
| 11 |
-
from src.app_helpers.pipeline_table import filter_auto_pipeline_table
|
| 12 |
|
| 13 |
|
| 14 |
class AutoPipelineResultsTests(unittest.TestCase):
|
|
@@ -138,6 +138,15 @@ class AutoPipelineResultsTests(unittest.TestCase):
|
|
| 138 |
|
| 139 |
|
| 140 |
class AutoPipelineFilterTests(unittest.TestCase):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
def test_empty_pipeline_selection_returns_no_rows(self):
|
| 142 |
update = filter_auto_pipeline_table(
|
| 143 |
self._sample_df(),
|
|
|
|
| 8 |
|
| 9 |
from src.leaderboard.read_auto_pipeline_results import get_auto_pipeline_results_df
|
| 10 |
from src.leaderboard.read_auto_pipeline_results import AUTO_PIPELINE_SEARCH_COL, AUTO_PIPELINE_SORT_COL
|
| 11 |
+
from src.app_helpers.pipeline_table import embed_dl_icon_in_hf_model_cell, filter_auto_pipeline_table
|
| 12 |
|
| 13 |
|
| 14 |
class AutoPipelineResultsTests(unittest.TestCase):
|
|
|
|
| 138 |
|
| 139 |
|
| 140 |
class AutoPipelineFilterTests(unittest.TestCase):
|
| 141 |
+
def test_download_icon_is_embedded_in_hf_model_cell(self):
|
| 142 |
+
cell = '<a target="_blank" href="https://huggingface.co/INC4AI/Qwen3-W4A16">INC4AI/Qwen3-W4A16</a>'
|
| 143 |
+
|
| 144 |
+
result = embed_dl_icon_in_hf_model_cell(cell)
|
| 145 |
+
|
| 146 |
+
self.assertIn('class="dl-cell-link"', result)
|
| 147 |
+
self.assertIn('data-repo="INC4AI/Qwen3-W4A16"', result)
|
| 148 |
+
self.assertIn('title="INC4AI/Qwen3-W4A16"', result)
|
| 149 |
+
|
| 150 |
def test_empty_pipeline_selection_returns_no_rows(self):
|
| 151 |
update = filter_auto_pipeline_table(
|
| 152 |
self._sample_df(),
|
tests/test_ci_dispatcher.py
CHANGED
|
@@ -70,8 +70,14 @@ def _old_str(hours_ago):
|
|
| 70 |
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 71 |
|
| 72 |
|
| 73 |
-
def _make_dispatcher(status_path, requests_path=None, max_concurrent=4, pat="fake-pat"
|
| 74 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
repo = MagicMock()
|
| 76 |
repo.active_branch.name = "main"
|
| 77 |
repo.working_dir = os.path.dirname(status_path)
|
|
@@ -87,6 +93,7 @@ def _make_dispatcher(status_path, requests_path=None, max_concurrent=4, pat="fak
|
|
| 87 |
requests_path=requests_path or status_path,
|
| 88 |
azure_pat=pat,
|
| 89 |
max_concurrent=max_concurrent,
|
|
|
|
| 90 |
)
|
| 91 |
|
| 92 |
|
|
@@ -516,6 +523,73 @@ class TestReconcileActiveRuns:
|
|
| 516 |
d._repo.remotes.origin.push.assert_not_called()
|
| 517 |
|
| 518 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 519 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 520 |
# 5. _collect_status_summary
|
| 521 |
# βββββββββββββββββββββββββββββββββββββββοΏ½οΏ½οΏ½βββββββββββββββββββββββββββββββ
|
|
|
|
| 70 |
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
| 71 |
|
| 72 |
|
| 73 |
+
def _make_dispatcher(status_path, requests_path=None, max_concurrent=4, pat="fake-pat",
|
| 74 |
+
enable_retry=True):
|
| 75 |
+
"""Build a CIDispatcher with a fully-mocked repo.
|
| 76 |
+
|
| 77 |
+
``enable_retry`` defaults to ``True`` because most reconcile tests assert
|
| 78 |
+
the retry path (failure β Pending β Failed). Tests for the retry-disabled
|
| 79 |
+
behaviour (failure β terminal Quant/Eval Failed) pass ``enable_retry=False``.
|
| 80 |
+
"""
|
| 81 |
repo = MagicMock()
|
| 82 |
repo.active_branch.name = "main"
|
| 83 |
repo.working_dir = os.path.dirname(status_path)
|
|
|
|
| 93 |
requests_path=requests_path or status_path,
|
| 94 |
azure_pat=pat,
|
| 95 |
max_concurrent=max_concurrent,
|
| 96 |
+
enable_retry=enable_retry,
|
| 97 |
)
|
| 98 |
|
| 99 |
|
|
|
|
| 523 |
d._repo.remotes.origin.push.assert_not_called()
|
| 524 |
|
| 525 |
|
| 526 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 527 |
+
# 4b. _reconcile_active_runs with retry DISABLED (terminal failed statuses)
|
| 528 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 529 |
+
|
| 530 |
+
class TestReconcileNoRetry:
|
| 531 |
+
"""With enable_retry=False, Azure failures map to terminal failed statuses
|
| 532 |
+
(Quant Failed / Eval Failed) instead of being reset to Pending."""
|
| 533 |
+
|
| 534 |
+
def setup_method(self):
|
| 535 |
+
self._tmpdir = tempfile.mkdtemp(prefix="test_noretry_")
|
| 536 |
+
self._status_dir = os.path.join(self._tmpdir, "status")
|
| 537 |
+
os.makedirs(self._status_dir)
|
| 538 |
+
|
| 539 |
+
def _dispatcher(self):
|
| 540 |
+
d = _make_dispatcher(self._status_dir, enable_retry=False)
|
| 541 |
+
d._repo.working_dir = self._tmpdir
|
| 542 |
+
return d
|
| 543 |
+
|
| 544 |
+
def test_quant_failed_on_azure_failure(self):
|
| 545 |
+
fp = _write_status(self._status_dir, "org", "a.json", {
|
| 546 |
+
"model": "org/a", "status": "Running", "script": "auto_quant",
|
| 547 |
+
"ci_run_id": 10, "triggered_time": _now_str(),
|
| 548 |
+
})
|
| 549 |
+
d = self._dispatcher()
|
| 550 |
+
resp = _mock_azure_response(200, {"state": "completed", "result": "failed"})
|
| 551 |
+
with patch("src.ci_dispatcher.http_requests.get", return_value=resp):
|
| 552 |
+
d._reconcile_active_runs()
|
| 553 |
+
data = _read_status(fp)
|
| 554 |
+
assert data["status"] == "Quant Failed"
|
| 555 |
+
assert data["ci_run_id"] == 10 # preserved for debugging
|
| 556 |
+
assert "retry_count" not in data # no retry bookkeeping
|
| 557 |
+
|
| 558 |
+
def test_eval_failed_on_azure_canceled(self):
|
| 559 |
+
fp = _write_status(self._status_dir, "org", "b.json", {
|
| 560 |
+
"model": "org/b", "status": "Triggered", "script": "auto_eval",
|
| 561 |
+
"ci_run_id": 20, "triggered_time": _now_str(),
|
| 562 |
+
})
|
| 563 |
+
d = self._dispatcher()
|
| 564 |
+
resp = _mock_azure_response(200, {"state": "completed", "result": "canceled"})
|
| 565 |
+
with patch("src.ci_dispatcher.http_requests.get", return_value=resp):
|
| 566 |
+
d._reconcile_active_runs()
|
| 567 |
+
assert _read_status(fp)["status"] == "Eval Failed"
|
| 568 |
+
|
| 569 |
+
def test_in_progress_still_updates_to_running(self):
|
| 570 |
+
"""Non-failure reconciliation (inProgress β Running) works regardless."""
|
| 571 |
+
fp = _write_status(self._status_dir, "org", "c.json", {
|
| 572 |
+
"model": "org/c", "status": "Triggered", "script": "auto_quant",
|
| 573 |
+
"ci_run_id": 30, "triggered_time": _now_str(),
|
| 574 |
+
})
|
| 575 |
+
d = self._dispatcher()
|
| 576 |
+
resp = _mock_azure_response(200, {"state": "inProgress", "result": ""})
|
| 577 |
+
with patch("src.ci_dispatcher.http_requests.get", return_value=resp):
|
| 578 |
+
d._reconcile_active_runs()
|
| 579 |
+
assert _read_status(fp)["status"] == "Running"
|
| 580 |
+
|
| 581 |
+
def test_succeeded_stays_running_within_grace(self):
|
| 582 |
+
fp = _write_status(self._status_dir, "org", "d.json", {
|
| 583 |
+
"model": "org/d", "status": "Running", "script": "auto_quant",
|
| 584 |
+
"ci_run_id": 40, "triggered_time": _now_str(),
|
| 585 |
+
})
|
| 586 |
+
d = self._dispatcher()
|
| 587 |
+
resp = _mock_azure_response(200, {"state": "completed", "result": "succeeded"})
|
| 588 |
+
with patch("src.ci_dispatcher.http_requests.get", return_value=resp):
|
| 589 |
+
d._reconcile_active_runs()
|
| 590 |
+
assert _read_status(fp)["status"] == "Running"
|
| 591 |
+
|
| 592 |
+
|
| 593 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 594 |
# 5. _collect_status_summary
|
| 595 |
# βββββββββββββββββββββββββββββββββββββββοΏ½οΏ½οΏ½βββββββββββββββββββββββββββββββ
|
tests/test_eval_queue.py
CHANGED
|
@@ -436,6 +436,109 @@ def test_submitted_quant_is_failed_when_result_is_newer():
|
|
| 436 |
shutil.rmtree(results_dir, ignore_errors=True)
|
| 437 |
|
| 438 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 439 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 440 |
# Test 6: Synthetic β unknown status entries are silently dropped
|
| 441 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββοΏ½οΏ½οΏ½ββββββββ
|
|
|
|
| 436 |
shutil.rmtree(results_dir, ignore_errors=True)
|
| 437 |
|
| 438 |
|
| 439 |
+
def test_resubmitted_quant_pending_removes_old_failed_entry():
|
| 440 |
+
"""A later accepted quant re-submit should remove the old failed row."""
|
| 441 |
+
tmpdir = tempfile.mkdtemp(prefix="test_queue_resubmit_pending_quant_")
|
| 442 |
+
results_dir = tempfile.mkdtemp(prefix="test_results_resubmit_pending_quant_")
|
| 443 |
+
try:
|
| 444 |
+
failed_entry = {
|
| 445 |
+
"model": "org/model",
|
| 446 |
+
"revision": "main",
|
| 447 |
+
"private": False,
|
| 448 |
+
"quant_scheme": "INT4 (W4A16)",
|
| 449 |
+
"quant_precision": "4bit",
|
| 450 |
+
"quant_weight_dtype": "int4",
|
| 451 |
+
"status": "Pending",
|
| 452 |
+
"submitted_time": "2026-05-21T08:00:00Z",
|
| 453 |
+
"script": "auto_quant",
|
| 454 |
+
"model_params": 7.0,
|
| 455 |
+
}
|
| 456 |
+
pending_entry = dict(failed_entry, status="Pending", submitted_time="2026-05-21T10:00:00Z")
|
| 457 |
+
with open(os.path.join(tmpdir, "failed_request.json"), "w") as fh:
|
| 458 |
+
json.dump(failed_entry, fh)
|
| 459 |
+
with open(os.path.join(tmpdir, "pending_request.json"), "w") as fh:
|
| 460 |
+
json.dump(pending_entry, fh)
|
| 461 |
+
|
| 462 |
+
failed_result = {
|
| 463 |
+
"model_id": "org/model",
|
| 464 |
+
"generated_at": "2026-05-21T09:00:00Z",
|
| 465 |
+
"run_dir": "runs/failed",
|
| 466 |
+
"copied_files": ["x"],
|
| 467 |
+
"quant_summary": {"scheme": "W4A16", "status": "failed"},
|
| 468 |
+
"accuracy": {"status": "missing"},
|
| 469 |
+
}
|
| 470 |
+
with open(os.path.join(results_dir, "results_failed.json"), "w") as fh:
|
| 471 |
+
json.dump(failed_result, fh)
|
| 472 |
+
|
| 473 |
+
finished_df, running_df, pending_df, failed_df = get_evaluation_queue_df(
|
| 474 |
+
tmpdir, QUANT_COLS, request_type="quant", results_path=results_dir
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
+
assert len(finished_df) == 0
|
| 478 |
+
assert len(pending_df) == 1
|
| 479 |
+
assert len(failed_df) == 0
|
| 480 |
+
assert pending_df["model"].astype(str).str.contains("org/model", regex=False).any()
|
| 481 |
+
finally:
|
| 482 |
+
shutil.rmtree(tmpdir, ignore_errors=True)
|
| 483 |
+
shutil.rmtree(results_dir, ignore_errors=True)
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
def test_resubmitted_quant_success_removes_old_failed_entry():
|
| 487 |
+
"""A later successful quant re-submit should remove the old failed row."""
|
| 488 |
+
tmpdir = tempfile.mkdtemp(prefix="test_queue_resolved_quant_")
|
| 489 |
+
results_dir = tempfile.mkdtemp(prefix="test_results_resolved_quant_")
|
| 490 |
+
try:
|
| 491 |
+
failed_entry = {
|
| 492 |
+
"model": "org/model",
|
| 493 |
+
"revision": "main",
|
| 494 |
+
"private": False,
|
| 495 |
+
"quant_scheme": "INT4 (W4A16)",
|
| 496 |
+
"quant_precision": "4bit",
|
| 497 |
+
"quant_weight_dtype": "int4",
|
| 498 |
+
"status": "Pending",
|
| 499 |
+
"submitted_time": "2026-05-21T08:00:00Z",
|
| 500 |
+
"script": "auto_quant",
|
| 501 |
+
"model_params": 7.0,
|
| 502 |
+
}
|
| 503 |
+
success_entry = dict(failed_entry, status="Pending", submitted_time="2026-05-21T10:00:00Z")
|
| 504 |
+
with open(os.path.join(tmpdir, "failed_request.json"), "w") as fh:
|
| 505 |
+
json.dump(failed_entry, fh)
|
| 506 |
+
with open(os.path.join(tmpdir, "success_request.json"), "w") as fh:
|
| 507 |
+
json.dump(success_entry, fh)
|
| 508 |
+
|
| 509 |
+
failed_result = {
|
| 510 |
+
"model_id": "org/model",
|
| 511 |
+
"generated_at": "2026-05-21T09:00:00Z",
|
| 512 |
+
"run_dir": "runs/failed",
|
| 513 |
+
"copied_files": ["x"],
|
| 514 |
+
"quant_summary": {"scheme": "W4A16", "status": "failed"},
|
| 515 |
+
"accuracy": {"status": "missing"},
|
| 516 |
+
}
|
| 517 |
+
success_result = {
|
| 518 |
+
"model_id": "org/model",
|
| 519 |
+
"generated_at": "2026-05-21T11:00:00Z",
|
| 520 |
+
"run_dir": "runs/success",
|
| 521 |
+
"copied_files": ["x"],
|
| 522 |
+
"quant_summary": {"scheme": "W4A16", "status": "success"},
|
| 523 |
+
"accuracy": {"status": "success"},
|
| 524 |
+
}
|
| 525 |
+
with open(os.path.join(results_dir, "results_failed.json"), "w") as fh:
|
| 526 |
+
json.dump(failed_result, fh)
|
| 527 |
+
with open(os.path.join(results_dir, "results_success.json"), "w") as fh:
|
| 528 |
+
json.dump(success_result, fh)
|
| 529 |
+
|
| 530 |
+
finished_df, running_df, pending_df, failed_df = get_evaluation_queue_df(
|
| 531 |
+
tmpdir, QUANT_COLS, request_type="quant", results_path=results_dir
|
| 532 |
+
)
|
| 533 |
+
|
| 534 |
+
assert len(finished_df) == 1
|
| 535 |
+
assert len(failed_df) == 0
|
| 536 |
+
assert finished_df["model"].astype(str).str.contains("org/model", regex=False).any()
|
| 537 |
+
finally:
|
| 538 |
+
shutil.rmtree(tmpdir, ignore_errors=True)
|
| 539 |
+
shutil.rmtree(results_dir, ignore_errors=True)
|
| 540 |
+
|
| 541 |
+
|
| 542 |
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 543 |
# Test 6: Synthetic β unknown status entries are silently dropped
|
| 544 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββοΏ½οΏ½οΏ½ββββββββ
|
tests/test_submit.py
CHANGED
|
@@ -56,7 +56,7 @@ def _is_error(result: str) -> bool:
|
|
| 56 |
|
| 57 |
def _is_success(result: str) -> bool:
|
| 58 |
"""Check if result is a styled_message (green) response."""
|
| 59 |
-
return "color: green" in result
|
| 60 |
|
| 61 |
def _is_warning(result: str) -> bool:
|
| 62 |
"""Check if result is a styled_warning (orange) response."""
|
|
@@ -272,11 +272,12 @@ def test_add_new_quant_allows_whitelisted_resubmit_for_failed_entry(monkeypatch,
|
|
| 272 |
monkeypatch.setattr(submit_module, "compute_single_eta", lambda *_args, **_kwargs: 1)
|
| 273 |
monkeypatch.setattr(submit_module, "format_eta", lambda *_args, **_kwargs: "1h")
|
| 274 |
|
| 275 |
-
uploaded = {"called": False, "file_tag": None}
|
| 276 |
|
| 277 |
def _fake_upload(entry, user_name, model_path, file_tag, model, task_label="eval"):
|
| 278 |
uploaded["called"] = True
|
| 279 |
uploaded["file_tag"] = file_tag
|
|
|
|
| 280 |
|
| 281 |
monkeypatch.setattr(submit_module, "_upload_to_hub", _fake_upload)
|
| 282 |
|
|
@@ -286,10 +287,12 @@ def test_add_new_quant_allows_whitelisted_resubmit_for_failed_entry(monkeypatch,
|
|
| 286 |
private=False,
|
| 287 |
quant_scheme="INT4 (W4A16)",
|
| 288 |
submitted_by="alice",
|
|
|
|
| 289 |
))
|
| 290 |
|
| 291 |
assert uploaded["called"] is True
|
| 292 |
assert _is_success(result)
|
|
|
|
| 293 |
# Re-submission must not overwrite the previous failed status file: the
|
| 294 |
# filename gets a timestamp suffix appended to keep both records.
|
| 295 |
assert uploaded["file_tag"] is not None
|
|
@@ -327,6 +330,7 @@ def test_add_new_eval_allows_whitelisted_resubmit_for_failed_entry(monkeypatch,
|
|
| 327 |
monkeypatch.setattr(submit_module, "_SUBMITTER_DATES", {})
|
| 328 |
monkeypatch.setattr(submit_module, "_load_eval_cache", lambda: None)
|
| 329 |
monkeypatch.setattr(submit_module, "_common_pre_checks", lambda *_args, **_kwargs: None)
|
|
|
|
| 330 |
monkeypatch.setattr(
|
| 331 |
submit_module,
|
| 332 |
"is_model_on_hub",
|
|
@@ -360,11 +364,12 @@ def test_add_new_eval_allows_whitelisted_resubmit_for_failed_entry(monkeypatch,
|
|
| 360 |
monkeypatch.setattr(submit_module, "compute_single_eta", lambda *_args, **_kwargs: 1)
|
| 361 |
monkeypatch.setattr(submit_module, "format_eta", lambda *_args, **_kwargs: "1h")
|
| 362 |
|
| 363 |
-
uploaded = {"called": False, "file_tag": None}
|
| 364 |
|
| 365 |
def _fake_upload(entry, user_name, model_path, file_tag, model, task_label="eval"):
|
| 366 |
uploaded["called"] = True
|
| 367 |
uploaded["file_tag"] = file_tag
|
|
|
|
| 368 |
|
| 369 |
monkeypatch.setattr(submit_module, "_upload_to_hub", _fake_upload)
|
| 370 |
|
|
@@ -374,10 +379,12 @@ def test_add_new_eval_allows_whitelisted_resubmit_for_failed_entry(monkeypatch,
|
|
| 374 |
private=False,
|
| 375 |
compute_dtype="INT4 (W4A16)",
|
| 376 |
submitted_by="alice",
|
|
|
|
| 377 |
))
|
| 378 |
|
| 379 |
assert uploaded["called"] is True
|
| 380 |
assert _is_success(result)
|
|
|
|
| 381 |
# Re-submission must not overwrite the previous failed status file: the
|
| 382 |
# filename gets a timestamp suffix appended to keep both records.
|
| 383 |
assert uploaded["file_tag"] is not None
|
|
|
|
| 56 |
|
| 57 |
def _is_success(result: str) -> bool:
|
| 58 |
"""Check if result is a styled_message (green) response."""
|
| 59 |
+
return "color: green" in result or "#f0fdf4" in result
|
| 60 |
|
| 61 |
def _is_warning(result: str) -> bool:
|
| 62 |
"""Check if result is a styled_warning (orange) response."""
|
|
|
|
| 272 |
monkeypatch.setattr(submit_module, "compute_single_eta", lambda *_args, **_kwargs: 1)
|
| 273 |
monkeypatch.setattr(submit_module, "format_eta", lambda *_args, **_kwargs: "1h")
|
| 274 |
|
| 275 |
+
uploaded = {"called": False, "file_tag": None, "entry": None}
|
| 276 |
|
| 277 |
def _fake_upload(entry, user_name, model_path, file_tag, model, task_label="eval"):
|
| 278 |
uploaded["called"] = True
|
| 279 |
uploaded["file_tag"] = file_tag
|
| 280 |
+
uploaded["entry"] = entry
|
| 281 |
|
| 282 |
monkeypatch.setattr(submit_module, "_upload_to_hub", _fake_upload)
|
| 283 |
|
|
|
|
| 287 |
private=False,
|
| 288 |
quant_scheme="INT4 (W4A16)",
|
| 289 |
submitted_by="alice",
|
| 290 |
+
submitted_orgs=["intel", "research"],
|
| 291 |
))
|
| 292 |
|
| 293 |
assert uploaded["called"] is True
|
| 294 |
assert _is_success(result)
|
| 295 |
+
assert uploaded["entry"]["submitted_orgs"] == ["intel", "research"]
|
| 296 |
# Re-submission must not overwrite the previous failed status file: the
|
| 297 |
# filename gets a timestamp suffix appended to keep both records.
|
| 298 |
assert uploaded["file_tag"] is not None
|
|
|
|
| 330 |
monkeypatch.setattr(submit_module, "_SUBMITTER_DATES", {})
|
| 331 |
monkeypatch.setattr(submit_module, "_load_eval_cache", lambda: None)
|
| 332 |
monkeypatch.setattr(submit_module, "_common_pre_checks", lambda *_args, **_kwargs: None)
|
| 333 |
+
monkeypatch.setattr(submit_module, "is_gguf_on_hub", lambda *_args, **_kwargs: (False, "", None, None))
|
| 334 |
monkeypatch.setattr(
|
| 335 |
submit_module,
|
| 336 |
"is_model_on_hub",
|
|
|
|
| 364 |
monkeypatch.setattr(submit_module, "compute_single_eta", lambda *_args, **_kwargs: 1)
|
| 365 |
monkeypatch.setattr(submit_module, "format_eta", lambda *_args, **_kwargs: "1h")
|
| 366 |
|
| 367 |
+
uploaded = {"called": False, "file_tag": None, "entry": None}
|
| 368 |
|
| 369 |
def _fake_upload(entry, user_name, model_path, file_tag, model, task_label="eval"):
|
| 370 |
uploaded["called"] = True
|
| 371 |
uploaded["file_tag"] = file_tag
|
| 372 |
+
uploaded["entry"] = entry
|
| 373 |
|
| 374 |
monkeypatch.setattr(submit_module, "_upload_to_hub", _fake_upload)
|
| 375 |
|
|
|
|
| 379 |
private=False,
|
| 380 |
compute_dtype="INT4 (W4A16)",
|
| 381 |
submitted_by="alice",
|
| 382 |
+
submitted_orgs=["intel", "research"],
|
| 383 |
))
|
| 384 |
|
| 385 |
assert uploaded["called"] is True
|
| 386 |
assert _is_success(result)
|
| 387 |
+
assert uploaded["entry"]["submitted_orgs"] == ["intel", "research"]
|
| 388 |
# Re-submission must not overwrite the previous failed status file: the
|
| 389 |
# filename gets a timestamp suffix appended to keep both records.
|
| 390 |
assert uploaded["file_tag"] is not None
|