rigelbar's picture
Implement P1 review productivity layer
2b5291f
Raw
History Blame Contribute Delete
12.6 kB
from __future__ import annotations
from copy import deepcopy
from .constants import APPROVAL_REQUIRED_STAGE_IDS
from .compat import model_to_dict, model_validate
from .models import (
ArtifactStatus,
IssueSeverity,
IssueStatus,
IssueType,
LayoutSpec,
PedagogicalRole,
ProposedChange,
ProposedChangeSet,
ReviewRole,
RevisionConstraints,
Slide,
SlideClaim,
SpeakerNotes,
VisualAsset,
)
from .quality import approve_current_artifact, create_artifact_version, grade_stage, now_iso, upsert_issue
from .workflow import build_empty_state, record_prompt_run
def valid_minimal_job():
state = build_empty_state(
deck_title="Intro to Deterministic Slide QA",
source_url="mock://source/course-notes",
template_url="mock://template/course",
dry_run=True,
)
state.source_chunks = {
"chunk_1": "Deterministic slide QA requires objective mapping and supported claims.",
"chunk_2": "A worked example helps learners apply the concept.",
}
state.objectives = {
"obj_1": "Explain deterministic slide quality checks.",
"obj_2": "Apply quality checks to a worked example.",
}
state.slides = {
"slide_1": Slide(
slide_id="slide_1",
slide_number=1,
title="Deterministic Slide QA",
visible_text="Quality checks make slide readiness explicit.",
bullet_points=["Trace objectives", "Check evidence", "Validate layout"],
objective_ids=["obj_1"],
pedagogical_role=PedagogicalRole.CONCEPT,
requires_visual=True,
speaker_notes=SpeakerNotes(
slide_id="slide_1",
notes_text="Explain why readiness must be explicit before export.",
instructor_intent="Connect quality gates to production trust.",
estimated_teaching_time_seconds=180,
),
),
"slide_2": Slide(
slide_id="slide_2",
slide_number=2,
title="Worked QA Example",
visible_text="Use the checklist to evaluate one generated slide.",
bullet_points=["Map objective", "Inspect claim support", "Confirm layout"],
objective_ids=["obj_2"],
pedagogical_role=PedagogicalRole.WORKED_EXAMPLE,
requires_visual=False,
speaker_notes=SpeakerNotes(
slide_id="slide_2",
notes_text="Walk through each checklist item and ask learners what blocks export.",
instructor_intent="Make the gating rule concrete.",
estimated_teaching_time_seconds=240,
),
),
}
state.claims = {
"claim_1": SlideClaim(
claim_id="claim_1",
slide_id="slide_1",
claim_text="Quality checks make slide readiness explicit.",
source_ids=["source_1"],
source_chunk_ids=["chunk_1"],
review_status="supported",
),
"claim_2": SlideClaim(
claim_id="claim_2",
slide_id="slide_2",
claim_text="A worked example helps learners apply the concept.",
source_ids=["source_1"],
source_chunk_ids=["chunk_2"],
review_status="supported",
),
}
state.visual_assets = {
"asset_1": VisualAsset(
asset_id="asset_1",
slide_id="slide_1",
asset_type="diagram",
path_or_url="mock://asset/qa-flow",
prompt="Simple quality gate diagram",
purpose="instructional",
alt_text="Flow from generation through grading, approval, and export.",
source="mock",
license_status="generated",
approved_for_export=True,
)
}
state.layout_specs = {
"slide_1": LayoutSpec(
slide_id="slide_1",
layout_id="title_bullets_visual",
approved_template_id="default_course_template",
slot_assignments={
"title": "Deterministic Slide QA",
"bullets": ["Trace objectives", "Check evidence", "Validate layout"],
"visual": "asset_1",
},
),
"slide_2": LayoutSpec(
slide_id="slide_2",
layout_id="worked_example",
approved_template_id="default_course_template",
slot_assignments={
"title": "Worked QA Example",
"problem": "Evaluate one generated slide.",
"steps": ["Map objective", "Inspect claim support", "Confirm layout"],
},
),
}
# Pydantic validates nested dicts assigned above during this explicit round-trip;
# this explicit round-trip keeps fixture construction terse and typed.
state = model_validate(type(state), model_to_dict(state))
for stage_id in APPROVAL_REQUIRED_STAGE_IDS:
prompt_run = record_prompt_run(
state,
stage_id,
rendered_prompt=f"Fixture generation for {stage_id}",
)
artifact = create_artifact_version(
state,
stage_id,
{"fixture": stage_id},
created_by="mock",
status=ArtifactStatus.CANDIDATE,
prompt_run_id=prompt_run.prompt_run_id,
mark_downstream_stale=False,
)
prompt_run.output_artifact_version_id = artifact.artifact_version_id
grade_stage(stage_id, state)
approve_current_artifact(state, stage_id, reviewer_name="fixture_reviewer")
return state
def missing_objective_mapping_job():
state = deepcopy(valid_minimal_job())
state.slides["slide_2"].objective_ids = []
return state
def stale_downstream_job():
state = deepcopy(valid_minimal_job())
create_artifact_version(
state,
"slide_outline_order",
{"fixture": "changed outline"},
created_by="human",
status=ArtifactStatus.CANDIDATE,
mark_downstream_stale=True,
)
return state
def invalidated_approval_job():
state = deepcopy(valid_minimal_job())
create_artifact_version(
state,
"text_generation",
{"fixture": "human edit after approval"},
created_by="human",
status=ArtifactStatus.CANDIDATE,
mark_downstream_stale=False,
)
return state
def unsupported_claim_job():
state = deepcopy(valid_minimal_job())
claim = state.claims["claim_1"]
claim.review_status = "unsupported"
claim.source_ids = []
claim.source_chunk_ids = []
return state
def missing_visual_asset_job():
state = deepcopy(valid_minimal_job())
state.visual_assets = {}
return state
def invalid_layout_job():
state = deepcopy(valid_minimal_job())
state.layout_specs["slide_1"] = LayoutSpec(
slide_id="slide_1",
layout_id="raw_coordinates",
approved_template_id=None,
slot_assignments={"x": 10, "y": 20, "width": 400, "height": 300},
)
return state
def text_density_failure_job():
state = deepcopy(valid_minimal_job())
state.slides["slide_1"].visible_text = " ".join(["dense"] * 80)
state.slides["slide_1"].bullet_points = ["one", "two", "three", "four", "five"]
return state
def review_queue_mixed_issues_job():
state = deepcopy(valid_minimal_job())
upsert_issue(
state,
IssueType.TEXT_DENSITY_EXCEEDED,
IssueSeverity.MAJOR,
"Slide text is too dense.",
stage_id="text_generation",
slide_id="slide_1",
)
upsert_issue(
state,
IssueType.ALT_TEXT_MISSING,
IssueSeverity.MINOR,
"Alt text needs review.",
stage_id="aesthetic_review",
slide_id="slide_1",
)
blocker = upsert_issue(
state,
IssueType.UNSUPPORTED_CLAIM,
IssueSeverity.BLOCKER,
"Unsupported claim blocks export.",
stage_id="technical_review",
slide_id="slide_1",
claim_id="claim_1",
)
blocker.assigned_role = ReviewRole.SME
return state
def role_filtered_review_job():
state = review_queue_mixed_issues_job()
visual_issue = upsert_issue(
state,
IssueType.LAYOUT_SCHEMA_INVALID,
IssueSeverity.MAJOR,
"Layout needs visual design review.",
stage_id="aesthetic_ordering_visual_composition",
slide_id="slide_2",
)
visual_issue.assigned_role = ReviewRole.VISUAL_DESIGNER
return state
def waivable_major_issue_job():
state = deepcopy(valid_minimal_job())
upsert_issue(
state,
IssueType.TEXT_DENSITY_EXCEEDED,
IssueSeverity.MAJOR,
"Major density issue can be waived with rationale.",
stage_id="text_generation",
slide_id="slide_1",
)
return state
def non_waivable_blocker_job():
state = deepcopy(valid_minimal_job())
upsert_issue(
state,
IssueType.UNSUPPORTED_CLAIM,
IssueSeverity.BLOCKER,
"Blocker cannot be waived.",
stage_id="technical_review",
slide_id="slide_1",
claim_id="claim_1",
)
return state
def proposed_change_set_job():
from .review import critique_artifact_for_improvement, create_proposed_change_set
state = waivable_major_issue_job()
critique = critique_artifact_for_improvement("text_generation", state)
create_proposed_change_set("text_generation", state, critique=critique)
return state
def constraint_violation_change_set_job():
state = deepcopy(valid_minimal_job())
artifact = next(
artifact
for artifact in state.artifacts.values()
if artifact.stage_id == "title_generation" and artifact.is_current
)
change_set = ProposedChangeSet(
change_set_id="changes_constraint_violation",
stage_id="title_generation",
artifact_version_id=artifact.artifact_version_id,
created_at=now_iso(),
constraints=RevisionConstraints(preserve_slide_titles=True),
changes=[
ProposedChange(
change_id="change_title_violation",
target_type="slide_title",
target_id="slide_1",
field_path="title",
before=state.slides["slide_1"].title,
after="Changed title",
)
],
)
state.proposed_change_sets[change_set.change_set_id] = change_set
return state
def selected_slide_improvement_job():
return deepcopy(valid_minimal_job())
def fast_path_stops_at_blocker_job():
return non_waivable_blocker_job()
def semantic_diff_changed_slide_job():
state = deepcopy(valid_minimal_job())
first = create_artifact_version(
state,
"text_generation",
{"slides": [model_to_dict(slide) for slide in state.slides.values()]},
created_by="mock",
status=ArtifactStatus.CANDIDATE,
mark_downstream_stale=False,
)
state.slides["slide_1"].title = "Changed Semantic Title"
second = create_artifact_version(
state,
"text_generation",
{"slides": [model_to_dict(slide) for slide in state.slides.values()]},
created_by="mock",
status=ArtifactStatus.CANDIDATE,
mark_downstream_stale=False,
)
first.status = ArtifactStatus.CANDIDATE
state.artifacts[first.artifact_version_id] = first
state.artifacts[second.artifact_version_id] = second
return state
def review_packet_job():
state = review_queue_mixed_issues_job()
issue = next(issue for issue in state.issues.values() if issue.severity == IssueSeverity.MAJOR)
issue.status = IssueStatus.WAIVED
issue.waiver_reason = "Accepted for this pilot deck."
return state
def restore_candidate_version_job():
state = deepcopy(valid_minimal_job())
artifact = create_artifact_version(
state,
"text_generation",
{"fixture": "restore source"},
created_by="human",
status=ArtifactStatus.CANDIDATE,
mark_downstream_stale=False,
)
artifact.is_current = False
artifact.status = ArtifactStatus.CANDIDATE
current = create_artifact_version(
state,
"text_generation",
{"fixture": "current candidate"},
created_by="human",
status=ArtifactStatus.CANDIDATE,
mark_downstream_stale=False,
)
artifact.status = ArtifactStatus.CANDIDATE
artifact.is_current = False
state.artifacts[artifact.artifact_version_id] = artifact
state.artifacts[current.artifact_version_id] = current
return state