Spaces:
Sleeping
Sleeping
File size: 59,311 Bytes
c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 2b5291f c5d7500 22e4fbd c5d7500 22e4fbd c5d7500 22e4fbd c5d7500 22e4fbd c5d7500 22e4fbd c5d7500 22e4fbd c5d7500 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 | from __future__ import annotations
import difflib
import hashlib
import json
import re
from datetime import UTC, datetime
from typing import Any
from .constants import (
APPROVAL_REQUIRED_STAGE_IDS,
APPROVED_LAYOUTS,
INSTRUCTIONAL_ROLES,
RAW_COORDINATE_KEYS,
STAGE_IDS,
STAGE_RUBRICS,
TEXT_DENSITY_LIMITS,
)
from .models import (
ArtifactStatus,
ArtifactVersion,
AuditEvent,
DeckHealthSummary,
ExportPreflightReport,
IssueSeverity,
IssueStatus,
IssueType,
LayoutSpec,
ObjectiveTrace,
PipelineState,
QualityIssue,
ReviewRole,
RubricDimensionScore,
SlideQAStatus,
StageApproval,
StageGradeResult,
)
PASSING_SCORE = 80
SLIDE_REVIEW_STAGE_IDS = {
"text_generation",
"image_visual_asset_generation",
"aesthetic_ordering_visual_composition",
"technical_review",
"pedagogical_review",
"aesthetic_review",
}
def now_iso() -> str:
return datetime.now(tz=UTC).replace(microsecond=0).isoformat()
def clamp_score(score: float) -> int:
return max(0, min(100, int(round(score))))
def stable_hash(content: Any) -> str:
rendered = json.dumps(content, sort_keys=True, default=str)
return hashlib.sha256(rendered.encode("utf-8")).hexdigest()
def record_audit(
state: PipelineState,
event_type: str,
*,
stage_id: str | None = None,
slide_id: str | None = None,
objective_id: str | None = None,
artifact_version_id: str | None = None,
issue_id: str | None = None,
claim_id: str | None = None,
reason: str | None = None,
metadata: dict[str, Any] | None = None,
) -> AuditEvent:
event = AuditEvent(
event_id=f"evt_{len(state.audit_events) + 1:05d}",
event_type=event_type,
timestamp=now_iso(),
stage_id=stage_id,
slide_id=slide_id,
objective_id=objective_id,
artifact_version_id=artifact_version_id,
issue_id=issue_id,
claim_id=claim_id,
reason=reason,
metadata=metadata or {},
)
state.audit_events.append(event)
return event
def aggregate_rubric_score(rubric_scores: list[RubricDimensionScore]) -> int:
if not rubric_scores:
return 0
total_weight = sum(max(0.0, item.weight) for item in rubric_scores)
if total_weight <= 0:
return 0
weighted = sum(clamp_score(item.score) * max(0.0, item.weight) for item in rubric_scores)
return clamp_score(weighted / total_weight)
def issue_id_for(
issue_type: IssueType,
*,
stage_id: str | None = None,
slide_id: str | None = None,
objective_id: str | None = None,
claim_id: str | None = None,
artifact_version_id: str | None = None,
) -> str:
parts = [
issue_type.value,
stage_id or "deck",
slide_id or "all_slides",
objective_id or "all_objectives",
claim_id or "all_claims",
artifact_version_id or "current",
]
safe = [re.sub(r"[^a-zA-Z0-9_.-]+", "_", part) for part in parts]
return "issue:" + ":".join(safe)
def upsert_issue(
state: PipelineState,
issue_type: IssueType,
severity: IssueSeverity,
message: str,
*,
stage_id: str | None = None,
slide_id: str | None = None,
objective_id: str | None = None,
artifact_version_id: str | None = None,
claim_id: str | None = None,
suggested_fix: str | None = None,
) -> QualityIssue:
issue_id = issue_id_for(
issue_type,
stage_id=stage_id,
slide_id=slide_id,
objective_id=objective_id,
claim_id=claim_id,
artifact_version_id=artifact_version_id,
)
existing = state.issues.get(issue_id)
priority = {
IssueSeverity.BLOCKER: "critical",
IssueSeverity.MAJOR: "high",
IssueSeverity.MINOR: "medium",
IssueSeverity.INFO: "low",
}[severity]
if existing is None:
issue = QualityIssue(
issue_id=issue_id,
issue_type=issue_type,
severity=severity,
message=message,
stage_id=stage_id,
slide_id=slide_id,
objective_id=objective_id,
artifact_version_id=artifact_version_id,
claim_id=claim_id,
suggested_fix=suggested_fix,
priority=priority,
last_seen_artifact_version_id=artifact_version_id,
created_at=now_iso(),
)
state.issues[issue_id] = issue
record_audit(
state,
"issue_created",
stage_id=stage_id,
slide_id=slide_id,
objective_id=objective_id,
artifact_version_id=artifact_version_id,
issue_id=issue_id,
claim_id=claim_id,
reason=message,
)
return issue
existing.severity = severity
existing.message = message
existing.suggested_fix = suggested_fix
existing.resolved = False
existing.status = IssueStatus.OPEN
existing.priority = priority
existing.resolved_at = None
existing.resolution_note = None
existing.last_seen_artifact_version_id = artifact_version_id
return existing
def resolve_issue(state: PipelineState, issue_id: str) -> None:
issue = state.issues.get(issue_id)
if issue and not issue.resolved:
issue.resolved = True
issue.status = IssueStatus.RESOLVED
issue.resolved_at = now_iso()
record_audit(state, "issue_resolved", issue_id=issue_id, reason="Condition cleared")
def _issue_blocks_quality(issue: QualityIssue) -> bool:
if issue.resolved or issue.status == IssueStatus.RESOLVED:
return False
if issue.status == IssueStatus.WAIVED:
return issue.severity == IssueSeverity.BLOCKER
if issue.status == IssueStatus.WONT_FIX:
return issue.severity == IssueSeverity.BLOCKER
return True
def unresolved_issues(state: PipelineState) -> list[QualityIssue]:
return [issue for issue in state.issues.values() if _issue_blocks_quality(issue)]
def unresolved_stage_issues(stage_id: str, state: PipelineState) -> list[QualityIssue]:
return [issue for issue in unresolved_issues(state) if issue.stage_id == stage_id]
def has_unresolved_blockers(stage_id: str, state: PipelineState) -> bool:
return any(
issue.severity == IssueSeverity.BLOCKER for issue in unresolved_stage_issues(stage_id, state)
)
def has_unresolved_blockers_anywhere(state: PipelineState) -> bool:
return any(issue.severity == IssueSeverity.BLOCKER for issue in unresolved_issues(state))
def get_current_stage_artifact(stage_id: str, state: PipelineState) -> ArtifactVersion | None:
stage = state.stages.get(stage_id)
if stage and stage.current_artifact_version_id:
artifact = state.artifacts.get(stage.current_artifact_version_id)
if artifact and artifact.is_current:
return artifact
for artifact_id in reversed(state.stage_artifact_versions.get(stage_id, [])):
artifact = state.artifacts.get(artifact_id)
if artifact and artifact.is_current:
return artifact
return None
def create_artifact_version(
state: PipelineState,
stage_id: str,
content: Any,
*,
created_by: str,
status: ArtifactStatus = ArtifactStatus.CANDIDATE,
prompt_run_id: str | None = None,
mark_downstream_stale: bool = True,
) -> ArtifactVersion:
previous = get_current_stage_artifact(stage_id, state)
if previous:
previous.is_current = False
if previous.status in {ArtifactStatus.DRAFT, ArtifactStatus.CANDIDATE}:
previous.status = ArtifactStatus.INVALIDATED
version_number = len(state.stage_artifact_versions.get(stage_id, [])) + 1
artifact_version_id = f"{stage_id}_v{version_number}"
artifact = ArtifactVersion(
artifact_version_id=artifact_version_id,
artifact_id=stage_id,
stage_id=stage_id,
version_number=version_number,
created_at=now_iso(),
created_by=created_by, # type: ignore[arg-type]
content_hash=stable_hash(content),
status=status,
parent_artifact_version_ids=[previous.artifact_version_id] if previous else [],
upstream_stage_versions={
upstream_id: upstream.current_artifact_version_id
for upstream_id, upstream in state.stages.items()
if STAGE_IDS.index(upstream_id) < STAGE_IDS.index(stage_id)
and upstream.current_artifact_version_id
},
prompt_run_id=prompt_run_id,
metadata={"content": content},
)
state.artifacts[artifact_version_id] = artifact
state.stage_artifact_versions.setdefault(stage_id, []).append(artifact_version_id)
state.stages[stage_id].current_artifact_version_id = artifact_version_id
state.stages[stage_id].is_stale = False
for issue in list(state.issues.values()):
if issue.stage_id == stage_id and issue.issue_type == IssueType.STALE_DOWNSTREAM_STAGE:
resolve_issue(state, issue.issue_id)
record_audit(
state,
"artifact_promoted" if status == ArtifactStatus.APPROVED else "artifact_created",
stage_id=stage_id,
artifact_version_id=artifact_version_id,
metadata={"status": status.value, "created_by": created_by},
)
invalidate_approval_for_stage(state, stage_id, "Artifact changed after approval")
if mark_downstream_stale:
mark_downstream_stages_stale(stage_id, state)
return artifact
def invalidate_approval_for_stage(state: PipelineState, stage_id: str, reason: str) -> None:
for approval in state.approvals:
if approval.stage_id != stage_id or approval.approval_status != "approved":
continue
approval.approval_status = "invalidated"
approval.invalidated_at = now_iso()
approval.invalidation_reason = reason
upsert_issue(
state,
IssueType.APPROVAL_INVALIDATED,
IssueSeverity.BLOCKER,
f"Approval for {stage_id} was invalidated: {reason}",
stage_id=stage_id,
artifact_version_id=approval.artifact_version_id,
suggested_fix="Re-grade and approve the current artifact version.",
)
record_audit(
state,
"approval_invalidated",
stage_id=stage_id,
artifact_version_id=approval.artifact_version_id,
reason=reason,
)
def has_valid_human_approval(stage_id: str, state: PipelineState) -> bool:
artifact = get_current_stage_artifact(stage_id, state)
if artifact is None:
return False
if artifact.status not in {ArtifactStatus.APPROVED, ArtifactStatus.EXPORTED}:
return False
return any(
approval.stage_id == stage_id
and approval.artifact_version_id == artifact.artifact_version_id
and approval.approval_status == "approved"
for approval in state.approvals
)
def approve_current_artifact(
state: PipelineState,
stage_id: str,
reviewer_name: str = "human_reviewer",
reviewer_role: ReviewRole | None = None,
) -> StageApproval:
artifact = get_current_stage_artifact(stage_id, state)
if artifact is None:
raise ValueError(f"Cannot approve {stage_id}: current artifact is missing.")
artifact.status = ArtifactStatus.APPROVED
approval = StageApproval(
stage_id=stage_id,
artifact_version_id=artifact.artifact_version_id,
reviewer_name=reviewer_name or "human_reviewer",
reviewer_role=reviewer_role,
approved_at=now_iso(),
)
state.approvals.append(approval)
record_audit(
state,
"approval_created",
stage_id=stage_id,
artifact_version_id=artifact.artifact_version_id,
metadata={
"reviewer_name": approval.reviewer_name,
"reviewer_role": approval.reviewer_role.value if approval.reviewer_role else None,
},
)
record_audit(
state,
"artifact_promoted",
stage_id=stage_id,
artifact_version_id=artifact.artifact_version_id,
metadata={"status": artifact.status.value},
)
return approval
def mark_downstream_stages_stale(changed_stage_id: str, state: PipelineState) -> PipelineState:
if changed_stage_id not in STAGE_IDS:
return state
changed_index = STAGE_IDS.index(changed_stage_id)
for stage_id in STAGE_IDS[changed_index + 1 :]:
stage = state.stages[stage_id]
stage.is_stale = True
artifact = get_current_stage_artifact(stage_id, state)
artifact_version_id = None
if artifact is not None:
artifact.status = ArtifactStatus.STALE
artifact_version_id = artifact.artifact_version_id
record_audit(
state,
"artifact_marked_stale",
stage_id=stage_id,
artifact_version_id=artifact.artifact_version_id,
reason=f"Upstream stage {changed_stage_id} changed",
)
invalidate_approval_for_stage(
state,
stage_id,
f"Upstream stage {changed_stage_id} changed",
)
upsert_issue(
state,
IssueType.STALE_DOWNSTREAM_STAGE,
IssueSeverity.BLOCKER,
f"{stage_id} is stale because {changed_stage_id} changed.",
stage_id=stage_id,
artifact_version_id=artifact_version_id,
suggested_fix="Regenerate or re-grade this stage against the current upstream artifacts.",
)
record_audit(
state,
"stage_marked_stale",
stage_id=stage_id,
reason=f"Upstream stage {changed_stage_id} changed",
)
return state
def can_unlock_next_stage(stage_id: str, state: PipelineState) -> bool:
stage = state.stages[stage_id]
if stage.score is None or stage.score < PASSING_SCORE:
return False
if has_unresolved_blockers(stage_id, state):
return False
if stage.is_stale:
return False
if not has_valid_human_approval(stage_id, state):
return False
current_artifact = get_current_stage_artifact(stage_id, state)
if current_artifact is None:
return False
return current_artifact.status in {ArtifactStatus.APPROVED, ArtifactStatus.EXPORTED}
def get_stage_lock_reasons(stage_id: str, state: PipelineState) -> list[str]:
reasons: list[str] = []
stage = state.stages[stage_id]
if stage.is_stale:
reasons.append("Stage is stale.")
if stage_id == STAGE_IDS[0]:
return reasons
previous_stage_id = STAGE_IDS[STAGE_IDS.index(stage_id) - 1]
previous = state.stages[previous_stage_id]
if previous.score is None:
reasons.append("Previous stage has not been scored.")
elif previous.score < PASSING_SCORE:
reasons.append("Previous stage score is below 80.")
if has_unresolved_blockers(previous_stage_id, state):
reasons.append("Previous stage has unresolved blockers.")
previous_artifact = get_current_stage_artifact(previous_stage_id, state)
if previous_artifact is None:
reasons.append("Current artifact is missing.")
elif previous_artifact.status not in {ArtifactStatus.APPROVED, ArtifactStatus.EXPORTED}:
reasons.append("Current artifact is not approved.")
if not has_valid_human_approval(previous_stage_id, state):
invalidated = any(
approval.stage_id == previous_stage_id
and approval.approval_status == "invalidated"
for approval in state.approvals
)
reasons.append("Human approval was invalidated." if invalidated else "Human approval is missing.")
if previous.is_stale:
reasons.append("Previous stage is stale.")
return list(dict.fromkeys(reasons))
def _visible_words(text: str) -> int:
return len(re.findall(r"\b[\w'-]+\b", text))
def _slide_visible_text(slide_id: str, state: PipelineState) -> str:
slide = state.slides[slide_id]
return " ".join([slide.title or "", slide.visible_text, " ".join(slide.bullet_points)]).strip()
def compute_objective_traces(state: PipelineState) -> list[ObjectiveTrace]:
traces: list[ObjectiveTrace] = []
for objective_id, objective_text in state.objectives.items():
mapped = [
slide.slide_id
for slide in state.slides.values()
if objective_id in slide.objective_ids
]
evidence = [
state.slides[slide_id].title or state.slides[slide_id].visible_text[:80]
for slide_id in mapped
]
issue_ids: list[str] = []
if not mapped:
coverage_score = 0
coverage_status = "uncovered"
issue = upsert_issue(
state,
IssueType.OBJECTIVE_UNCOVERED,
IssueSeverity.BLOCKER,
f"Learning objective {objective_id} is not mapped to any slide.",
stage_id="slide_outline_order",
objective_id=objective_id,
suggested_fix="Map this objective to at least one slide.",
)
issue_ids.append(issue.issue_id)
else:
explicit_scores = [
state.slides[slide_id].objective_coverage_scores.get(objective_id)
for slide_id in mapped
if objective_id in state.slides[slide_id].objective_coverage_scores
]
if explicit_scores:
coverage_score = clamp_score(sum(explicit_scores) / len(explicit_scores))
elif len(mapped) >= 2:
coverage_score = 90
else:
coverage_score = 75
if coverage_score < 50:
coverage_status = "weak"
issue = upsert_issue(
state,
IssueType.OBJECTIVE_WEAKLY_COVERED,
IssueSeverity.MAJOR,
f"Learning objective {objective_id} is only weakly covered.",
stage_id="slide_outline_order",
objective_id=objective_id,
suggested_fix="Add stronger evidence or another mapped slide.",
)
issue_ids.append(issue.issue_id)
elif coverage_score < 80:
coverage_status = "partial"
else:
coverage_status = "strong"
trace = ObjectiveTrace(
objective_id=objective_id,
objective_text=objective_text,
mapped_slide_ids=mapped,
coverage_score=coverage_score,
coverage_status=coverage_status, # type: ignore[arg-type]
evidence=evidence,
issue_ids=issue_ids,
)
state.objective_traces[objective_id] = trace
traces.append(trace)
record_audit(
state,
"objective_trace_updated",
objective_id=objective_id,
metadata={"coverage_status": trace.coverage_status, "coverage_score": trace.coverage_score},
)
return traces
def validate_claim_support(state: PipelineState) -> list[QualityIssue]:
issues: list[QualityIssue] = []
for claim in state.claims.values():
unsupported = claim.review_status == "unsupported"
missing_support = not claim.source_ids and not claim.source_chunk_ids
if not unsupported and not missing_support:
continue
issue = upsert_issue(
state,
IssueType.UNSUPPORTED_CLAIM,
IssueSeverity.BLOCKER,
f"Claim {claim.claim_id} is not source-grounded.",
stage_id="technical_review",
slide_id=claim.slide_id,
claim_id=claim.claim_id,
suggested_fix="Attach source chunks or mark the claim supported after review.",
)
if issue.issue_id not in claim.issue_ids:
claim.issue_ids.append(issue.issue_id)
issues.append(issue)
return issues
def check_text_density(slide_id: str, state: PipelineState) -> list[QualityIssue]:
slide = state.slides[slide_id]
role = slide.pedagogical_role.value
limits = TEXT_DENSITY_LIMITS.get(role, TEXT_DENSITY_LIMITS["unknown"])
issues: list[QualityIssue] = []
visible_words = _visible_words(_slide_visible_text(slide_id, state))
if visible_words > limits["max_visible_words"]:
issues.append(
upsert_issue(
state,
IssueType.TEXT_DENSITY_EXCEEDED,
IssueSeverity.MAJOR,
f"Slide {slide_id} has {visible_words} visible words; limit is "
f"{limits['max_visible_words']} for role {role}.",
stage_id="text_generation",
slide_id=slide_id,
suggested_fix="Reduce visible text or move explanation into speaker notes.",
)
)
if len(slide.bullet_points) > limits["max_bullets"]:
issues.append(
upsert_issue(
state,
IssueType.TEXT_DENSITY_EXCEEDED,
IssueSeverity.MAJOR,
f"Slide {slide_id} has {len(slide.bullet_points)} bullets; limit is "
f"{limits['max_bullets']} for role {role}.",
stage_id="text_generation",
slide_id=slide_id,
suggested_fix="Combine or remove bullets.",
)
)
concept_count = slide.distinct_concept_count if slide.distinct_concept_count is not None else 1
if len(slide.objective_ids) > 2 or concept_count > 3:
issues.append(
upsert_issue(
state,
IssueType.COGNITIVE_LOAD_HIGH,
IssueSeverity.MAJOR,
f"Slide {slide_id} carries too many objectives or concepts.",
stage_id="pedagogical_review",
slide_id=slide_id,
suggested_fix="Split the slide or narrow its instructional focus.",
)
)
return issues
def check_speaker_notes(
slide_id: str,
state: PipelineState,
*,
strict: bool = False,
) -> list[QualityIssue]:
slide = state.slides[slide_id]
if slide.pedagogical_role.value not in INSTRUCTIONAL_ROLES:
return []
notes_text = slide.speaker_notes.notes_text if slide.speaker_notes else None
if notes_text and notes_text.strip():
return []
severity = IssueSeverity.BLOCKER if strict else IssueSeverity.MAJOR
stage_id = "pedagogical_review" if strict else "text_generation"
return [
upsert_issue(
state,
IssueType.SPEAKER_NOTES_MISSING,
severity,
f"Instructional slide {slide_id} is missing speaker notes.",
stage_id=stage_id,
slide_id=slide_id,
suggested_fix="Add instructor intent and teaching notes for this slide.",
)
]
def _slot_text(value: Any) -> str:
if isinstance(value, list):
return " ".join(str(item) for item in value)
if isinstance(value, dict):
return json.dumps(value, sort_keys=True)
return str(value)
def _has_raw_coordinates(layout: LayoutSpec) -> bool:
if RAW_COORDINATE_KEYS & set(layout.slot_assignments):
return True
for value in layout.slot_assignments.values():
if isinstance(value, dict) and RAW_COORDINATE_KEYS & set(value):
return True
return False
def validate_layout_spec(layout: LayoutSpec, state: PipelineState) -> list[QualityIssue]:
issues: list[QualityIssue] = []
if _has_raw_coordinates(layout):
issues.append(
upsert_issue(
state,
IssueType.LAYOUT_SCHEMA_INVALID,
IssueSeverity.BLOCKER,
f"Slide {layout.slide_id} layout uses raw coordinates instead of template slots.",
stage_id="aesthetic_ordering_visual_composition",
slide_id=layout.slide_id,
suggested_fix="Use an approved layout_id and named template slots.",
)
)
return issues
registry_entry = APPROVED_LAYOUTS.get(layout.layout_id)
if registry_entry is None:
issues.append(
upsert_issue(
state,
IssueType.LAYOUT_SCHEMA_INVALID,
IssueSeverity.BLOCKER,
f"Slide {layout.slide_id} uses unknown layout_id {layout.layout_id}.",
stage_id="aesthetic_ordering_visual_composition",
slide_id=layout.slide_id,
suggested_fix="Choose an approved layout ID.",
)
)
return issues
required_slots = registry_entry["required_slots"]
optional_slots = registry_entry["optional_slots"]
slot_word_limits = registry_entry["slot_word_limits"]
used_slots = set(layout.slot_assignments)
missing = required_slots - used_slots # type: ignore[operator]
unknown = used_slots - required_slots - optional_slots # type: ignore[operator]
if missing:
issues.append(
upsert_issue(
state,
IssueType.LAYOUT_SLOT_VIOLATION,
IssueSeverity.BLOCKER,
f"Slide {layout.slide_id} is missing required layout slots: {sorted(missing)}.",
stage_id="aesthetic_ordering_visual_composition",
slide_id=layout.slide_id,
suggested_fix="Populate all required slots for the selected layout.",
)
)
if unknown:
issues.append(
upsert_issue(
state,
IssueType.LAYOUT_SLOT_VIOLATION,
IssueSeverity.MAJOR,
f"Slide {layout.slide_id} uses unknown slots: {sorted(unknown)}.",
stage_id="aesthetic_ordering_visual_composition",
slide_id=layout.slide_id,
suggested_fix="Remove arbitrary slots and use the template schema.",
)
)
for slot, limit in slot_word_limits.items(): # type: ignore[union-attr]
if slot not in layout.slot_assignments:
continue
word_count = _visible_words(_slot_text(layout.slot_assignments[slot]))
if word_count > limit:
issues.append(
upsert_issue(
state,
IssueType.LAYOUT_SLOT_VIOLATION,
IssueSeverity.MAJOR,
f"Slide {layout.slide_id} slot {slot} has {word_count} words; limit is {limit}.",
stage_id="aesthetic_ordering_visual_composition",
slide_id=layout.slide_id,
suggested_fix="Shorten text in this layout slot.",
)
)
return issues
def check_accessibility(slide_id: str, state: PipelineState) -> list[QualityIssue]:
issues: list[QualityIssue] = []
slide = state.slides[slide_id]
visual_assets = [asset for asset in state.visual_assets.values() if asset.slide_id == slide_id]
for asset in visual_assets:
meaningful = asset.purpose in {"instructional", "unknown"}
if meaningful and not (asset.alt_text and asset.alt_text.strip()):
issues.append(
upsert_issue(
state,
IssueType.ALT_TEXT_MISSING,
IssueSeverity.MAJOR,
f"Visual asset {asset.asset_id} on slide {slide_id} is missing alt text.",
stage_id="aesthetic_review",
slide_id=slide_id,
suggested_fix="Add concise alt text for meaningful visuals.",
)
)
has_visible_text = bool(_slide_visible_text(slide_id, state).strip())
has_notes = bool(slide.speaker_notes and slide.speaker_notes.notes_text)
if visual_assets and not has_visible_text and not has_notes:
issues.append(
upsert_issue(
state,
IssueType.ALT_TEXT_MISSING,
IssueSeverity.MAJOR,
f"Image-only slide {slide_id} lacks accessible text or speaker notes.",
stage_id="aesthetic_review",
slide_id=slide_id,
suggested_fix="Add accessible slide text or speaker notes.",
)
)
contrast = slide.metadata.get("contrast_ratio")
if isinstance(contrast, int | float) and contrast < 4.5:
issues.append(
upsert_issue(
state,
IssueType.ACCESSIBILITY_CONTRAST_RISK,
IssueSeverity.MAJOR,
f"Slide {slide_id} has a contrast risk below 4.5:1.",
stage_id="aesthetic_review",
slide_id=slide_id,
suggested_fix="Adjust foreground/background colors in the template metadata.",
)
)
return issues
def check_visual_assets(slide_id: str, state: PipelineState) -> list[QualityIssue]:
slide = state.slides[slide_id]
assets = [asset for asset in state.visual_assets.values() if asset.slide_id == slide_id]
issues: list[QualityIssue] = []
if slide.requires_visual and not assets:
issues.append(
upsert_issue(
state,
IssueType.VISUAL_ASSET_MISSING,
IssueSeverity.BLOCKER,
f"Slide {slide_id} requires a visual asset but none exists.",
stage_id="image_visual_asset_generation",
slide_id=slide_id,
suggested_fix="Generate, upload, or assign a visual asset.",
)
)
for asset in assets:
if asset.purpose == "unknown":
issues.append(
upsert_issue(
state,
IssueType.VISUAL_ASSET_PURPOSE_MISSING,
IssueSeverity.MAJOR,
f"Visual asset {asset.asset_id} has no instructional/decorative purpose.",
stage_id="image_visual_asset_generation",
slide_id=slide_id,
suggested_fix="Mark the asset purpose before export.",
)
)
if asset.purpose in {"instructional", "unknown"} and not asset.alt_text:
issues.append(
upsert_issue(
state,
IssueType.ALT_TEXT_MISSING,
IssueSeverity.MAJOR,
f"Meaningful visual asset {asset.asset_id} is missing alt text.",
stage_id="image_visual_asset_generation",
slide_id=slide_id,
suggested_fix="Add alt text.",
)
)
if asset.license_status in {"unknown", "needs_review"}:
issues.append(
upsert_issue(
state,
IssueType.COPYRIGHT_OR_LICENSE_RISK,
IssueSeverity.MAJOR,
f"Visual asset {asset.asset_id} has unresolved license status.",
stage_id="image_visual_asset_generation",
slide_id=slide_id,
suggested_fix="Use generated/user-provided assets or review the license.",
)
)
return issues
def compute_slide_status(slide_id: str, state: PipelineState) -> SlideQAStatus:
slide = state.slides[slide_id]
check_text_density(slide_id, state)
check_speaker_notes(slide_id, state)
check_visual_assets(slide_id, state)
check_accessibility(slide_id, state)
layout = state.layout_specs.get(slide_id)
if layout:
validate_layout_spec(layout, state)
issue_ids = [
issue.issue_id
for issue in unresolved_issues(state)
if issue.slide_id == slide_id
]
slide_issues = [state.issues[issue_id] for issue_id in issue_ids]
blocker_count = sum(issue.severity == IssueSeverity.BLOCKER for issue in slide_issues)
major_count = sum(issue.severity == IssueSeverity.MAJOR for issue in slide_issues)
aggregate_score = clamp_score(100 - blocker_count * 45 - major_count * 15)
technical_score = 0 if any(issue.issue_type == IssueType.UNSUPPORTED_CLAIM for issue in slide_issues) else 100
pedagogical_score = clamp_score(
100
- 20
* sum(
issue.issue_type
in {
IssueType.SPEAKER_NOTES_MISSING,
IssueType.COGNITIVE_LOAD_HIGH,
IssueType.TEXT_DENSITY_EXCEEDED,
}
for issue in slide_issues
)
)
aesthetic_score = clamp_score(
100
- 20
* sum(
issue.issue_type
in {
IssueType.LAYOUT_SCHEMA_INVALID,
IssueType.LAYOUT_SLOT_VIOLATION,
IssueType.ALT_TEXT_MISSING,
IssueType.ACCESSIBILITY_CONTRAST_RISK,
}
for issue in slide_issues
)
)
stale = any(state.stages[stage_id].is_stale for stage_id in SLIDE_REVIEW_STAGE_IDS)
if blocker_count:
status = "blocked"
elif stale:
status = "stale"
elif major_count or aggregate_score < PASSING_SCORE:
status = "needs_revision"
elif all(has_valid_human_approval(stage_id, state) for stage_id in ("technical_review", "pedagogical_review", "aesthetic_review")):
status = "approved"
else:
status = "needs_review"
qa_status = SlideQAStatus(
slide_id=slide_id,
slide_number=slide.slide_number,
title=slide.title,
pedagogical_role=slide.pedagogical_role,
objective_ids=slide.objective_ids,
claim_ids=[claim.claim_id for claim in state.claims.values() if claim.slide_id == slide_id],
visual_asset_ids=[asset.asset_id for asset in state.visual_assets.values() if asset.slide_id == slide_id],
layout_id=layout.layout_id if layout else None,
aggregate_score=aggregate_score,
technical_score=technical_score,
pedagogical_score=pedagogical_score,
aesthetic_score=aesthetic_score,
issue_ids=issue_ids,
stale=stale,
status=status, # type: ignore[arg-type]
)
state.slide_statuses[slide_id] = qa_status
record_audit(
state,
"slide_status_updated",
slide_id=slide_id,
metadata={"status": qa_status.status, "aggregate_score": qa_status.aggregate_score},
)
return qa_status
def _coverage_average(state: PipelineState) -> int:
if not state.objective_traces:
compute_objective_traces(state)
if not state.objective_traces:
return 0
return clamp_score(
sum(trace.coverage_score for trace in state.objective_traces.values())
/ len(state.objective_traces)
)
def _deduct_for_issue_types(
state: PipelineState,
issue_types: set[IssueType],
*,
stage_id: str | None = None,
) -> int:
issues = [
issue
for issue in unresolved_issues(state)
if issue.issue_type in issue_types and (stage_id is None or issue.stage_id == stage_id)
]
if any(issue.severity == IssueSeverity.BLOCKER for issue in issues):
return 0
penalty = sum(45 if issue.severity == IssueSeverity.BLOCKER else 15 for issue in issues)
return clamp_score(100 - penalty)
def _score_dimension(stage_id: str, dimension_id: str, state: PipelineState) -> tuple[int, str]:
if stage_id == "setup_inputs":
source_material_present = bool(state.source_url or state.source_chunks)
required = [state.deck_title, state.template_url, source_material_present]
if dimension_id == "required_inputs_present":
return (
100 if all(required) else 30,
"Deck title, template URL, and either uploaded material or a material URL are checked.",
)
if dimension_id == "url_validity":
urls = [url for url in [state.source_url, state.template_url] if url]
valid = all(url.startswith(("http://", "https://", "mock://")) for url in urls)
return (100 if valid else 40, "Provided material/template URLs must be explicit URLs.")
safe = state.dry_run or bool(state.output_folder_id)
if state.mutation_target_url and state.mutation_target_url in {state.source_url, state.template_url}:
safe = False
return (100 if safe else 0, "Render/export targets must be safe.")
if stage_id == "source_extraction_objective_mapping":
if dimension_id == "source_extraction_completeness":
return (100 if state.source_chunks else 40, "At least one source chunk is required.")
if dimension_id == "objective_clarity":
clear = state.objectives and all(_visible_words(text) >= 3 for text in state.objectives.values())
return (100 if clear else 45, "Objectives should be readable, specific statements.")
objective_texts = [text.strip().lower() for text in state.objectives.values()]
unique = len(objective_texts) == len(set(objective_texts))
return (100 if unique else 55, "Objectives should not duplicate one another.")
if stage_id == "slide_outline_order":
compute_objective_traces(state)
if dimension_id == "objective_coverage":
return (_coverage_average(state), "Coverage is aggregated from objective traces.")
if dimension_id == "logical_flow":
ordered = [slide.slide_number for slide in state.slides.values()]
roles_known = all(slide.pedagogical_role.value != "unknown" for slide in state.slides.values())
return (100 if ordered == sorted(ordered) and roles_known else 65, "Slide order and roles are checked.")
if dimension_id == "appropriate_slide_count":
upper = max(1, len(state.objectives) * 3 + 2)
ok = 1 <= len(state.slides) <= upper
return (100 if ok else 60, "Slide count should fit the number of objectives.")
titles = [slide.title for slide in state.slides.values() if slide.title]
unique = len(titles) == len(set(titles))
return (100 if unique else 65, "Slide titles are used as a redundancy signal.")
if stage_id == "title_generation":
titled = [slide for slide in state.slides.values() if slide.title and len(slide.title.split()) >= 2]
if dimension_id == "title_clarity":
return (100 if len(titled) == len(state.slides) and state.slides else 45, "Every slide needs a clear title.")
if dimension_id == "title_specificity":
specific = all(slide.title and _visible_words(slide.title) >= 2 for slide in state.slides.values())
return (100 if specific else 60, "Titles should be more specific than a section label.")
aligned = all(slide.objective_ids for slide in state.slides.values())
return (100 if aligned else 60, "Titles are expected on objective-mapped slides.")
if stage_id == "text_generation":
for slide_id in state.slides:
check_text_density(slide_id, state)
check_speaker_notes(slide_id, state)
validate_claim_support(state)
if dimension_id == "clarity":
populated = all(_slide_visible_text(slide_id, state) for slide_id in state.slides)
return (100 if populated else 45, "Slides should contain visible instructional text.")
if dimension_id == "text_density":
return (
_deduct_for_issue_types(
state,
{IssueType.TEXT_DENSITY_EXCEEDED, IssueType.COGNITIVE_LOAD_HIGH},
),
"Density and cognitive-load checks are deterministic.",
)
if dimension_id == "source_grounding":
return (_deduct_for_issue_types(state, {IssueType.UNSUPPORTED_CLAIM}), "Claims must be source grounded.")
return (
_deduct_for_issue_types(state, {IssueType.SPEAKER_NOTES_MISSING}),
"Instructional roles should have speaker notes.",
)
if stage_id == "image_visual_asset_generation":
for slide_id in state.slides:
check_visual_assets(slide_id, state)
if dimension_id == "asset_completeness":
return (_deduct_for_issue_types(state, {IssueType.VISUAL_ASSET_MISSING}), "Required visuals must exist.")
if dimension_id == "alt_text_presence":
return (_deduct_for_issue_types(state, {IssueType.ALT_TEXT_MISSING}), "Meaningful visuals need alt text.")
if dimension_id == "license_or_generation_metadata":
return (
_deduct_for_issue_types(state, {IssueType.COPYRIGHT_OR_LICENSE_RISK}),
"Visual sources and license metadata are checked.",
)
return (
_deduct_for_issue_types(state, {IssueType.VISUAL_ASSET_PURPOSE_MISSING}),
"Visual purpose metadata is checked.",
)
if stage_id == "aesthetic_ordering_visual_composition":
for layout in state.layout_specs.values():
validate_layout_spec(layout, state)
if dimension_id == "layout_schema_validity":
return (_deduct_for_issue_types(state, {IssueType.LAYOUT_SCHEMA_INVALID}), "Layout IDs must be approved.")
if dimension_id == "slot_compliance":
return (_deduct_for_issue_types(state, {IssueType.LAYOUT_SLOT_VIOLATION}), "Slots must fit layout schema.")
if dimension_id == "visual_hierarchy":
return (100 if state.layout_specs else 50, "Slides should have explicit layouts.")
return (_deduct_for_issue_types(state, {IssueType.TEXT_DENSITY_EXCEEDED}), "Readability uses density signals.")
if stage_id == "technical_review":
validate_claim_support(state)
if dimension_id == "unsupported_claim_detection":
return (_deduct_for_issue_types(state, {IssueType.UNSUPPORTED_CLAIM}), "Unsupported claims are blockers.")
if dimension_id == "technical_accuracy":
return (_deduct_for_issue_types(state, {IssueType.TECHNICAL_INACCURACY}), "Technical issues are tracked.")
return (100 if state.claims else 85, "Claims provide terminology review surface.")
if stage_id == "pedagogical_review":
compute_objective_traces(state)
for slide_id in state.slides:
check_text_density(slide_id, state)
check_speaker_notes(slide_id, state, strict=True)
if dimension_id == "objective_coverage":
return (_coverage_average(state), "Objective coverage is aggregated from traces.")
if dimension_id == "speaker_notes_teachability":
return (_deduct_for_issue_types(state, {IssueType.SPEAKER_NOTES_MISSING}), "Speaker notes are required.")
if dimension_id == "examples_or_applications":
has_example = any(
slide.pedagogical_role.value in {"worked_example", "knowledge_check"}
for slide in state.slides.values()
)
return (100 if has_example else 75, "At least one practice/application role is preferred.")
return (100 if state.slides else 0, "Slides should progress through an ordered sequence.")
if stage_id == "aesthetic_review":
for slide_id in state.slides:
check_accessibility(slide_id, state)
for layout in state.layout_specs.values():
validate_layout_spec(layout, state)
if dimension_id == "accessibility":
return (
_deduct_for_issue_types(
state,
{IssueType.ALT_TEXT_MISSING, IssueType.ACCESSIBILITY_CONTRAST_RISK},
),
"P0 accessibility checks alt text and contrast metadata only.",
)
if dimension_id == "brand_template_compliance":
compliant = all(layout.approved_template_id for layout in state.layout_specs.values())
return (100 if compliant and state.layout_specs else 65, "Layouts should bind to an approved template.")
if dimension_id == "layout_consistency":
known = all(layout.layout_id in APPROVED_LAYOUTS for layout in state.layout_specs.values())
return (100 if known and state.layout_specs else 45, "Layouts must be from the approved registry.")
return (_deduct_for_issue_types(state, {IssueType.TEXT_DENSITY_EXCEEDED}), "Readability uses density signals.")
if stage_id == "final_render_export":
report = run_export_preflight(state)
if dimension_id == "preflight_passed":
return (100 if report.can_export else 0, report.summary)
if dimension_id == "all_required_approvals_valid":
valid_count = sum(has_valid_human_approval(stage_id, state) for stage_id in APPROVAL_REQUIRED_STAGE_IDS)
return (
clamp_score(100 * valid_count / len(APPROVAL_REQUIRED_STAGE_IDS)),
"All stages 1-10 require valid approvals.",
)
if dimension_id == "no_stale_stages":
stale_count = sum(stage.is_stale for stage in state.stages.values())
return (100 if stale_count == 0 else 0, "Stale stages block export.")
unsafe = state.mutation_target_url and state.mutation_target_url in {state.source_url, state.template_url}
return (0 if unsafe else 100, "Export target safety is checked.")
if stage_id == "audit_log_version_history":
if dimension_id == "audit_events_present":
return (100 if state.audit_events else 50, "Audit events should exist.")
if dimension_id == "artifact_versions_traceable":
traceable = all(artifact.content_hash for artifact in state.artifacts.values())
return (100 if traceable and state.artifacts else 50, "Artifact hashes provide traceability.")
return (100 if state.reviewer_notes else 80, "Reviewer notes are tracked when present.")
return (80, "Default deterministic rubric score.")
def grade_stage(stage_id: str, state: PipelineState) -> StageGradeResult:
artifact = get_current_stage_artifact(stage_id, state)
rubric_scores: list[RubricDimensionScore] = []
for dimension_id, label, weight in STAGE_RUBRICS[stage_id]:
dimension_score, rationale = _score_dimension(stage_id, dimension_id, state)
rubric_scores.append(
RubricDimensionScore(
dimension_id=dimension_id,
label=label,
score=dimension_score,
weight=weight,
rationale=rationale,
)
)
record_audit(
state,
"rubric_score_created",
stage_id=stage_id,
artifact_version_id=artifact.artifact_version_id if artifact else None,
metadata={"dimension_id": dimension_id, "score": dimension_score},
)
score = aggregate_rubric_score(rubric_scores)
issue_ids = [issue.issue_id for issue in unresolved_stage_issues(stage_id, state)]
recommended_actions = [
issue.suggested_fix or issue.message
for issue in unresolved_stage_issues(stage_id, state)
if issue.severity in {IssueSeverity.MAJOR, IssueSeverity.BLOCKER}
]
result = StageGradeResult(
stage_id=stage_id,
artifact_version_id=artifact.artifact_version_id if artifact else None,
score=score,
passed_threshold=score >= PASSING_SCORE,
rubric_scores=rubric_scores,
issue_ids=issue_ids,
recommended_actions=list(dict.fromkeys(recommended_actions)),
graded_at=now_iso(),
)
state.stages[stage_id].score = score
state.stages[stage_id].grade_result = result
record_audit(
state,
"stage_grade_created",
stage_id=stage_id,
artifact_version_id=result.artifact_version_id,
metadata={"score": score, "passed_threshold": result.passed_threshold},
)
return result
def compute_deck_health_summary(state: PipelineState) -> DeckHealthSummary:
if state.objectives:
compute_objective_traces(state)
for slide_id in state.slides:
compute_slide_status(slide_id, state)
unresolved = unresolved_issues(state)
blockers = [issue for issue in unresolved if issue.severity == IssueSeverity.BLOCKER]
majors = [issue for issue in unresolved if issue.severity == IssueSeverity.MAJOR]
slide_scores = [
status.aggregate_score
for status in state.slide_statuses.values()
if status.aggregate_score is not None
]
invalidated_approvals = [
approval for approval in state.approvals if approval.approval_status == "invalidated"
]
unsupported_claims = [
issue for issue in unresolved if issue.issue_type == IssueType.UNSUPPORTED_CLAIM
]
objectives_strong = sum(
trace.coverage_status == "strong" for trace in state.objective_traces.values()
)
objectives_partial_or_weak = sum(
trace.coverage_status in {"partial", "weak"}
for trace in state.objective_traces.values()
)
objectives_uncovered = sum(
trace.coverage_status == "uncovered" for trace in state.objective_traces.values()
)
can_export_now = (
not blockers
and not invalidated_approvals
and not any(stage.is_stale for stage in state.stages.values())
and all(has_valid_human_approval(stage_id, state) for stage_id in APPROVAL_REQUIRED_STAGE_IDS)
)
return DeckHealthSummary(
job_id=state.job_id,
deck_title=state.deck_title,
approved_stage_count=sum(
has_valid_human_approval(stage_id, state) for stage_id in state.stages
),
total_stage_count=len(state.stages),
stale_stage_count=sum(stage.is_stale for stage in state.stages.values()),
invalidated_approval_count=len(invalidated_approvals),
unresolved_blocker_count=len(blockers),
unresolved_major_issue_count=len(majors),
slide_count=len(state.slides),
average_slide_score=(
clamp_score(sum(slide_scores) / len(slide_scores)) if slide_scores else None
),
objectives_total=len(state.objectives),
objectives_strong=objectives_strong,
objectives_partial_or_weak=objectives_partial_or_weak,
objectives_uncovered=objectives_uncovered,
unsupported_claim_count=len(unsupported_claims),
can_export=can_export_now,
top_blockers=[issue.message for issue in blockers[:5]],
)
def compute_artifact_diff(previous_content: str | dict, current_content: str | dict) -> str:
if isinstance(previous_content, dict):
previous = json.dumps(previous_content, sort_keys=True, indent=2).splitlines(keepends=True)
else:
previous = str(previous_content).splitlines(keepends=True)
if isinstance(current_content, dict):
current = json.dumps(current_content, sort_keys=True, indent=2).splitlines(keepends=True)
else:
current = str(current_content).splitlines(keepends=True)
diff = difflib.unified_diff(previous, current, fromfile="previous", tofile="current")
return "".join(diff)
def _preflight_blocker(
state: PipelineState,
message: str,
*,
stage_id: str = "final_render_export",
slide_id: str | None = None,
artifact_version_id: str | None = None,
suggested_fix: str | None = None,
) -> QualityIssue:
return upsert_issue(
state,
IssueType.EXPORT_PREFLIGHT_FAILED,
IssueSeverity.BLOCKER,
message,
stage_id=stage_id,
slide_id=slide_id,
artifact_version_id=artifact_version_id,
suggested_fix=suggested_fix,
)
def _check_preflight_prerequisites(state: PipelineState) -> None:
if not state.deck_title or not state.template_url or not (state.source_url or state.source_chunks):
upsert_issue(
state,
IssueType.MISSING_REQUIRED_INPUT,
IssueSeverity.BLOCKER,
"Deck title, template URL, and course material upload or URL are required.",
stage_id="setup_inputs",
suggested_fix="Complete setup inputs before export.",
)
for stage_id in APPROVAL_REQUIRED_STAGE_IDS:
stage = state.stages[stage_id]
artifact = get_current_stage_artifact(stage_id, state)
if stage.score is None or stage.score < PASSING_SCORE:
upsert_issue(
state,
IssueType.SCORE_BELOW_THRESHOLD,
IssueSeverity.BLOCKER,
f"{stage_id} must be scored at 80 or higher before export.",
stage_id=stage_id,
suggested_fix="Grade or improve this stage.",
)
if not has_valid_human_approval(stage_id, state):
upsert_issue(
state,
IssueType.HUMAN_APPROVAL_MISSING,
IssueSeverity.BLOCKER,
f"{stage_id} does not have valid approval for its current artifact.",
stage_id=stage_id,
artifact_version_id=artifact.artifact_version_id if artifact else None,
suggested_fix="Approve the current passing artifact.",
)
if artifact is None:
_preflight_blocker(
state,
f"{stage_id} has no current artifact.",
stage_id=stage_id,
suggested_fix="Generate this stage.",
)
elif artifact.status not in {ArtifactStatus.APPROVED, ArtifactStatus.EXPORTED}:
_preflight_blocker(
state,
f"{stage_id} current artifact is {artifact.status.value}; export requires approved artifacts.",
stage_id=stage_id,
artifact_version_id=artifact.artifact_version_id,
suggested_fix="Approve the current artifact.",
)
for stage_id, stage in state.stages.items():
if stage.is_stale:
upsert_issue(
state,
IssueType.STALE_DOWNSTREAM_STAGE,
IssueSeverity.BLOCKER,
f"{stage_id} is stale and cannot be exported.",
stage_id=stage_id,
suggested_fix="Regenerate or re-grade stale stages.",
)
def _check_preflight_slide_requirements(state: PipelineState) -> None:
if not state.slides:
_preflight_blocker(
state,
"Slide inventory is missing.",
suggested_fix="Generate a slide outline before export.",
)
return
for slide_id, slide in state.slides.items():
if not slide.title:
_preflight_blocker(
state,
f"Slide {slide_id} is missing a title.",
slide_id=slide_id,
suggested_fix="Generate or edit slide titles.",
)
if slide.pedagogical_role.value == "unknown":
_preflight_blocker(
state,
f"Slide {slide_id} is missing a pedagogical role.",
slide_id=slide_id,
suggested_fix="Assign a pedagogical role.",
)
check_speaker_notes(slide_id, state, strict=True)
check_text_density(slide_id, state)
check_visual_assets(slide_id, state)
check_accessibility(slide_id, state)
if slide_id not in state.layout_specs:
upsert_issue(
state,
IssueType.LAYOUT_SCHEMA_INVALID,
IssueSeverity.BLOCKER,
f"Slide {slide_id} is missing layout JSON.",
stage_id="aesthetic_ordering_visual_composition",
slide_id=slide_id,
suggested_fix="Assign an approved template-first layout.",
)
else:
validate_layout_spec(state.layout_specs[slide_id], state)
status = compute_slide_status(slide_id, state)
if status.aggregate_score is None or status.aggregate_score < PASSING_SCORE:
_preflight_blocker(
state,
f"Slide {slide_id} aggregate score is below 80.",
slide_id=slide_id,
suggested_fix="Resolve slide-level issues.",
)
def _check_preflight_render_safety(state: PipelineState) -> None:
if not state.dry_run and not state.output_folder_id:
upsert_issue(
state,
IssueType.MISSING_REQUIRED_INPUT,
IssueSeverity.BLOCKER,
"Output folder ID is required when dry-run mode is off.",
stage_id="setup_inputs",
suggested_fix="Provide an output folder ID or keep dry-run mode enabled.",
)
if state.production_export_requested and state.dry_run:
upsert_issue(
state,
IssueType.RENDER_SAFETY_VIOLATION,
IssueSeverity.BLOCKER,
"Dry-run mode cannot perform a production export.",
stage_id="final_render_export",
suggested_fix="Disable dry-run mode only after all export targets are safe.",
)
if state.mutation_target_url and state.mutation_target_url in {state.source_url, state.template_url}:
upsert_issue(
state,
IssueType.RENDER_SAFETY_VIOLATION,
IssueSeverity.BLOCKER,
"Source/template URLs cannot be used as mutation targets.",
stage_id="final_render_export",
suggested_fix="Use a separate output folder or generated deck target.",
)
def run_export_preflight(state: PipelineState) -> ExportPreflightReport:
record_audit(state, "preflight_run", stage_id="final_render_export")
_check_preflight_prerequisites(state)
if state.objectives:
compute_objective_traces(state)
else:
upsert_issue(
state,
IssueType.OBJECTIVE_UNCOVERED,
IssueSeverity.BLOCKER,
"No learning objectives exist.",
stage_id="source_extraction_objective_mapping",
suggested_fix="Extract or provide learning objectives.",
)
validate_claim_support(state)
_check_preflight_slide_requirements(state)
_check_preflight_render_safety(state)
blocking_issue_ids = [
issue.issue_id
for issue in unresolved_issues(state)
if issue.severity == IssueSeverity.BLOCKER
]
warning_issue_ids = [
issue.issue_id
for issue in unresolved_issues(state)
if issue.severity in {IssueSeverity.MAJOR, IssueSeverity.MINOR}
]
can_export = not blocking_issue_ids
summary = (
"Export preflight passed."
if can_export
else f"Export preflight failed with {len(blocking_issue_ids)} blocker(s)."
)
report = ExportPreflightReport(
can_export=can_export,
checked_at=now_iso(),
blocking_issue_ids=blocking_issue_ids,
warning_issue_ids=warning_issue_ids,
summary=summary,
)
state.export_preflight_report = report
record_audit(
state,
"preflight_passed" if can_export else "preflight_failed",
stage_id="final_render_export",
metadata={"blocking_issue_count": len(blocking_issue_ids)},
)
return report
def can_export(state: PipelineState) -> bool:
return run_export_preflight(state).can_export
|